mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-17 02:12:40 +00:00
Compare commits
6
Commits
v0.46.10.0
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9c2c911886 | ||
|
|
5ee88a6c23 | ||
|
|
f4b233e8e3 | ||
|
|
9118e6117e | ||
|
|
658ab936b8 | ||
|
|
5ef85ac9e3 |
@@ -1,17 +1,33 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.12.2",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": { "name": "Garry Tan", "url": "https://github.com/garrytan" },
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": ["memory", "knowledge-base", "mcp", "search", "agent", "brain", "pgvector"],
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/.agents/gbrain-launcher",
|
||||
"args": ["serve", "--surface", "starter", "--source-guard"],
|
||||
"args": [
|
||||
"serve",
|
||||
"--surface",
|
||||
"starter",
|
||||
"--source-guard"
|
||||
],
|
||||
"cwd": "${CLAUDE_PLUGIN_ROOT}"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.12.2",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": { "name": "Garry Tan", "url": "https://github.com/garrytan" },
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": ["memory", "knowledge-base", "mcp", "search", "agent", "brain", "pgvector"],
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": "./.codex-plugin/mcp.json",
|
||||
"interface": {
|
||||
@@ -15,7 +26,10 @@
|
||||
"longDescription": "GBrain wires a personal knowledge brain into every session: hybrid keyword+vector search, entity graph traversal, synthesis, and memory your agent writes itself — served on the starter MCP surface (the seven memory verbs plus the daily-driver brain ops). Bundles the curated brain-first skill set: setup (walks install + gbrain init), cold-start day-one brain filling, ingest, query, briefing, upgrade, and more. Requires the gbrain CLI (bun install -g github:garrytan/gbrain#latest-stable) and a brain (gbrain init); the bundled setup skill walks the rest. Unix (macOS/Linux) only.",
|
||||
"developerName": "Garry Tan",
|
||||
"category": "Productivity",
|
||||
"capabilities": ["Interactive", "Write"],
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/garrytan/gbrain",
|
||||
"defaultPrompt": [
|
||||
"Search my brain, recall context across sessions, and write new memory as we work"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-runbook-stamp: 0.46.10.0 -->
|
||||
<!-- gbrain-runbook-stamp: 0.46.12.2 -->
|
||||
<!-- This stamp must equal the VERSION file at every release; CI enforces it
|
||||
(scripts/check-bootstrap-tag.sh). `gbrain bootstrap status` compares it to
|
||||
the installed binary and warns on skew. -->
|
||||
|
||||
+248
@@ -2,6 +2,254 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.46.12.2] - 2026-08-16
|
||||
|
||||
**Your agent can now do over MCP what it could only do from the CLI.** An
|
||||
audit of the CLI-versus-MCP surface found the gap was never that MCP filtered
|
||||
tools out — it was CLI commands that never got an operation entry, so a
|
||||
connected agent hit "unknown tool" and fell back to shelling out. This wave
|
||||
closes that: eleven new tools, so an agent can record and resolve predictions,
|
||||
capture a quick note, check queue health, and read search/cache diagnostics
|
||||
without leaving the MCP session.
|
||||
|
||||
### Added
|
||||
- **Record predictions from your agent.** `takes_add`, `takes_update`,
|
||||
`takes_resolve`, and `takes_supersede` let a connected agent write to the
|
||||
predictions ledger, not just read it — record a bet, refine its weight,
|
||||
resolve it with evidence, or supersede a stale claim. Resolutions an agent
|
||||
makes are tagged distinctly from your own, so the calibration scorecard keeps
|
||||
agent-made and owner-made verdicts separable (`takes_scorecard` shows the
|
||||
count).
|
||||
- **`capture` over MCP.** The "just remember this" write is now an MCP tool,
|
||||
not CLI-only: it auto-derives a stable inbox slug, dedupes identical content,
|
||||
and is on the daily-driver (`starter`) surface so bundled connect flows can
|
||||
reach it. Prefer it for quick notes; `put_page` still handles full-control
|
||||
writes.
|
||||
- **Queue health from your agent.** `get_job_stats` surfaces the job queue's
|
||||
per-type rollup and health counts over MCP, including the "wedged queue"
|
||||
signal (a worker alive but claiming nothing while work waits) — the one jobs
|
||||
command that had no MCP equivalent.
|
||||
- **Retrieval + cache diagnostics over MCP.** `search_stats`, `search_modes`,
|
||||
`search_tune`, and `cache_stats` expose the same read-only dashboards as
|
||||
`gbrain search` / `gbrain cache stats`, so an agent asked to diagnose
|
||||
retrieval quality or cost no longer has to shell out. Tuning recommendations
|
||||
come back as paste-ready commands; applying them stays a deliberate local
|
||||
step.
|
||||
- **Content-quality triage over MCP.** `quarantine_list` shows hidden and
|
||||
flagged pages for review; scanning and clearing stay local.
|
||||
- **Migration state in `get_health`.** The health dashboard now reports
|
||||
pending, partial, and wedged host migrations, so a remote operator can spot
|
||||
a stuck migration without opening a shell on the host.
|
||||
- **Thin-client installs route the new tools automatically.** On a
|
||||
connect-to-a-remote-brain install, `gbrain takes …`, `gbrain search
|
||||
stats|modes|tune`, `gbrain jobs stats`, `gbrain cache stats`, and `gbrain
|
||||
quarantine list` now run against the brain host instead of failing.
|
||||
|
||||
### Changed
|
||||
- **`gbrain whoknows` now shows its full ranked view.** The expertise-routing
|
||||
command was reachable but rendered generic output; it now renders the ranked
|
||||
table with per-factor explanations it was always meant to show.
|
||||
- **The agent tool catalog is honest on every transport.** On a local
|
||||
(stdio) connection, tools gated off by the brain owner no longer appear in
|
||||
the tool list only to be refused when called — they're hidden until enabled,
|
||||
matching how remote connections already behaved.
|
||||
|
||||
### Fixed
|
||||
- **Predictions written over MCP are safe by construction.** The
|
||||
markdown-canonical takes store keeps its file writes contained to the right
|
||||
source's working tree, writes atomically, refuses input that could corrupt
|
||||
the on-page table, and treats a database mirror hiccup as recoverable rather
|
||||
than losing the write — the same protections the main page writer already
|
||||
had, now covering the new remote write path.
|
||||
|
||||
To take advantage of v0.46.12.2: reconnect your coding agent (or restart
|
||||
`gbrain serve`) so the new tools appear in its catalog. On the default `full`
|
||||
surface every new tool is available immediately; on `starter`, `capture` joins
|
||||
the daily-driver set. Nothing to configure — the tools are read-only or
|
||||
scope-and-fence-protected exactly like the operations they mirror.
|
||||
## [0.46.12.0] - 2026-08-16
|
||||
|
||||
**Every surface that could still steer you toward the retiring embedding
|
||||
provider now tells the truth.** The provider's hosted API ends 2026-09-04;
|
||||
v0.46.3.0 stopped the CLI from *acting* on a switch, but the discovery
|
||||
surfaces — help text, provider setup output, doctor hints, historical agent
|
||||
playbooks — still read like a recommendation. A downstream agent reading that
|
||||
copy recommended switching a brain ONTO the dying provider; this release makes
|
||||
that impossible.
|
||||
|
||||
### Changed
|
||||
|
||||
- **`gbrain ze-switch` is now a pure refusal/redirect shim.** `--help` leads
|
||||
with RETIRED, the sunset date, and the one maintained off-ramp
|
||||
(`gbrain migrate embeddings --to voyage:voyage-4 --dim 1024`), and it now
|
||||
actually reaches you through the compiled binary (the generic help
|
||||
short-circuit used to hide the command's own help entirely). Every
|
||||
invocation refuses or redirects with exit 1; retired flags (`--resume`,
|
||||
`--non-interactive`, `--force`, …) are still accepted so old scripts get
|
||||
the refusal message instead of an unknown-flag error — even on a machine
|
||||
with no brain configured.
|
||||
- **Two scripted contracts changed deliberately:** `ze-switch --undo` no
|
||||
longer acts — it prints the exact `gbrain migrate embeddings` command that
|
||||
returns the brain to its pre-switch provider (the retired action wrote
|
||||
config the runtime never read and emptied vectors with no verified
|
||||
re-embed); and `ze-switch --dry-run --json` now returns
|
||||
`{status:'refused', reason:'provider_sunset'}` with exit 1 instead of a
|
||||
machine-readable plan targeting the dying provider (`status:'planned'`,
|
||||
exit 0). JSON envelopes carry both `…_preview` (cost preview) and live
|
||||
command fields, and every command those envelopes render preserves an
|
||||
explicit `--brain` selector so multi-brain setups are never pointed at the
|
||||
wrong database (the engine-free `--help` text shows the generic command).
|
||||
- **`gbrain providers env <sunsetting-provider>` replaces the signup funnel**
|
||||
(dashboard URL + get-a-key hint) with the deprecation notice, replacement
|
||||
models, and the migration command — key STATUS still renders for existing
|
||||
users. `providers explain` marks sunsetting rows with ⚠ instead of a green
|
||||
ready-check. Both render through one shared marker so the surfaces can't
|
||||
drift, and the behavior is generic: any future provider sunset inherits it.
|
||||
- **`gbrain doctor`'s missing-key hint is migration-first** on a sunsetting
|
||||
provider: the fix is the off-ramp; the key path survives as a secondary
|
||||
note for the remaining hosted window.
|
||||
- **Historical migration playbooks can no longer be followed past their
|
||||
banners.** The two switch-era skill files now open with "Do not execute
|
||||
any command in this file", their imperative recommendations are rewritten
|
||||
as past-tense record, and their frontmatter pitches are marked HISTORICAL.
|
||||
Provider docs drop the price-comparison sell copy and retitle setup as
|
||||
"existing brains and self-hosters only — do not onboard".
|
||||
|
||||
### Fixed
|
||||
|
||||
- The undo guidance is exact: a snapshot with reranking disabled but a model
|
||||
id still set now yields `--reranker off` (the old precedence would have
|
||||
re-enabled a reranker the pre-switch brain had off); nested model ids
|
||||
(`ollama:model:tag`, `openrouter:org/model`) validate correctly; snapshot
|
||||
fields are shape-checked before they land in a command you're told to run;
|
||||
a failed `--undo` never points back at `--undo`; and the three undo
|
||||
failure states (missing / invalid / unreadable snapshot) each report
|
||||
truthfully instead of claiming no switch was recorded.
|
||||
|
||||
### Removed
|
||||
|
||||
- The retired interactive switch banner and its benchmark pitch ("switch to
|
||||
the new provider — RECOMMENDED") no longer ship in the binary; the module
|
||||
that carried them is deleted ahead of the September removal.
|
||||
|
||||
### To take advantage of v0.46.12.0
|
||||
|
||||
`gbrain upgrade` is enough — no schema migration.
|
||||
|
||||
1. **Upgrade:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
2. **If your brain still embeds or reranks through the retiring provider**,
|
||||
run the off-ramp before 2026-09-04 (cost preview first):
|
||||
```bash
|
||||
gbrain migrate embeddings --to voyage:voyage-4 --dim 1024 --dry-run
|
||||
gbrain migrate embeddings --to voyage:voyage-4 --dim 1024
|
||||
```
|
||||
Your agent can follow `skills/migrations/v0.46.3.0.md` end to end.
|
||||
3. **Things to watch:** scripts that parsed `ze-switch --dry-run --json`'s
|
||||
old `planned` envelope or relied on `ze-switch --undo` acting in place
|
||||
must switch to `gbrain migrate embeddings` (the printed guidance names
|
||||
the exact command, including your `--brain` selector). If anything looks
|
||||
wrong, file an issue with `gbrain doctor` output:
|
||||
https://github.com/garrytan/gbrain/issues
|
||||
|
||||
## [0.46.11.0] - 2026-08-16
|
||||
|
||||
**Five operational failures from live production brains, fixed at the root.**
|
||||
A backlink auto-fix that could corrupt a page's frontmatter, a job queue that
|
||||
grew a multi-thousand-job backlog with no admission control and no alarm,
|
||||
junk filenames that imported as plausible-looking pages and polluted search,
|
||||
a read/write source-scoping asymmetry that misrouted pages in multi-source
|
||||
brains, and frontmatter types that silently filed into unexpected
|
||||
directories. Each fix ships with its regression pinned and a discovery
|
||||
surface so the same failure can't build up silently again.
|
||||
|
||||
### Added
|
||||
- **Queue admission control for background agents.** Identical parentless
|
||||
`subagent` submits now coalesce onto the existing waiting job (same owner
|
||||
lane, payload, and execution options — the response carries `coalesced:
|
||||
true` so callers can tell); jobs still waiting after 48 hours are cancelled
|
||||
with an auditable reason instead of queueing forever (`gbrain config set
|
||||
minions.ttl_waiting_hours.<name> <hours|0>` to tune or disable); and an
|
||||
optional per-type waiting quota (`minions.quota_max_waiting.<name>`,
|
||||
off by default) rejects new submits with a structured, retryable error once
|
||||
a backlog cap is hit — exact even under concurrent submitters. Everything
|
||||
disables at once with `GBRAIN_MINIONS_ADMISSION=0`.
|
||||
- **Warn-before-act for the new waiting-TTL.** The first sweep never fires
|
||||
cold: the worker (and `gbrain upgrade`) print a one-time notice with the
|
||||
affected-job count, then hold a one-hour grace window before the first
|
||||
cancellation so there's real time to tune or opt out.
|
||||
- **Divergent-queue alarms.** `gbrain jobs stats` gains Drained/Waiting
|
||||
columns, a per-type `DIVERGENT QUEUE` scream when intake structurally
|
||||
exceeds completions (with the exact config command to cap it), a
|
||||
waiting-TTL 24h cancellation line, and a `--json` document; `gbrain
|
||||
doctor`'s queue health check surfaces the same findings for cron
|
||||
topologies. TTL cancellations are never counted as useful drain.
|
||||
- **Stored-type visibility.** Sync and import now warn once per run when
|
||||
explicit frontmatter types are aliases or undeclared in the active schema
|
||||
pack (aggregated counts ride the sync result and the `--json` envelope for
|
||||
worker topologies; silence with `schema.type_warnings false`), and `gbrain
|
||||
schema lint` gains two data-plane rules that catch the existing corpus,
|
||||
scoped per source.
|
||||
- **`gbrain quarantine clear --source-id`** — clearing a slug that exists in
|
||||
multiple sources now errors with the source list instead of picking one
|
||||
arbitrarily.
|
||||
- **`malformed_path_pages` doctor check** — finds previously ingested pages
|
||||
backed by junk filenames and says exactly which are sweepable versus which
|
||||
need a rename.
|
||||
- A shared atomic file writer (`src/core/atomic-write.ts`): unique temp
|
||||
sibling, full-write loop, fsync, on-disk verification callback, mode
|
||||
preservation past the umask, and parent-directory fsync after the rename.
|
||||
|
||||
### Fixed
|
||||
- **`check-backlinks fix` can no longer corrupt frontmatter.** The timeline
|
||||
inserter now computes the body offset from the canonical frontmatter
|
||||
parser (never matching headings inside YAML), validates the page before
|
||||
and after the edit, writes atomically with an on-disk verify, takes the
|
||||
per-page lock, and isolates per-file errors so one bad page can't poison a
|
||||
batch. Pages with pre-existing broken frontmatter are skipped and reported
|
||||
instead of made worse.
|
||||
- **Junk filenames no longer import.** Markdown paths containing brackets or
|
||||
any path containing control characters are rejected at sync, import, and
|
||||
the direct file-import defense (before any slug is minted), with the skip
|
||||
visibly reported on every route — including dry runs, directory imports,
|
||||
and syncs whose only changes were malformed files. Previously ingested
|
||||
junk rows are swept by the next full sync; legitimately bracket-named
|
||||
markdown from older releases is preserved (rename to re-import), and
|
||||
code-strategy sources keep indexing framework layouts like `app/[id]/`.
|
||||
- **Source-scoped reads now mirror their writes.** The existence-check/write
|
||||
asymmetry that misrouted pages in multi-source brains is closed across the
|
||||
writer transaction (pages, links, raw data, validators, slug registry),
|
||||
file import, image import, code reindex, and integrity repair — enforced
|
||||
going forward by a CI guard, with unscoped reads made deterministic
|
||||
(default source first) in both engines.
|
||||
- Waiting-TTL cancellations flow through the canonical cancel path so
|
||||
aggregator parents always resolve, reasons stamp only the jobs that
|
||||
actually expired, and a cancelled child frees its idempotency slot.
|
||||
- Interactive `gbrain agent run` prints `coalesced` (with the matched job id)
|
||||
instead of a false `submitted` when admission coalescing matched an
|
||||
existing waiting job; the remote submit surface returns the same signal and
|
||||
maps quota rejections to a structured `rate_limited` error.
|
||||
- Job names and frontmatter-derived type strings are sanitized before
|
||||
terminal output, and copy-pasteable remediation hints only embed values
|
||||
that are shell-safe tokens.
|
||||
- Page-lock acquisition is now exclusive-create, so two processes reclaiming
|
||||
a stale lock can no longer both proceed and lose one side's writes.
|
||||
- The advisor's stalled-jobs recommendation and the schema-lint retype hint
|
||||
now point at commands that exist.
|
||||
|
||||
### To take advantage of v0.46.11.0
|
||||
Upgrade and restart the worker (`bun install -g github:garrytan/gbrain#latest-stable
|
||||
&& gbrain upgrade`). The one-time waiting-TTL notice will print with your
|
||||
affected-job count and hold a one-hour grace window — tune with `gbrain
|
||||
config set minions.ttl_waiting_hours.subagent <hours|0>` before the first
|
||||
sweep if 48h isn't right for you. Then check `gbrain jobs stats`: if you see
|
||||
a `DIVERGENT QUEUE` scream, the printed `minions.quota_max_waiting.<name>`
|
||||
command is the opt-in cap. Existing junk-filename pages are removed by your
|
||||
next full `gbrain sync` (files stay on disk; rename a file to re-import its
|
||||
content), and `gbrain doctor` will name anything that needs a manual rename.
|
||||
|
||||
## [0.46.10.0] - 2026-08-16
|
||||
|
||||
**Switching embedding and reranking providers is now one guess-free
|
||||
|
||||
@@ -58,7 +58,12 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
sites; `ctx.remote !== false` for untrust-unless-explicit-false). Don't default it falsy.
|
||||
- **Source isolation.** Every read-side op routes through `sourceScopeOpts(ctx)`; precedence
|
||||
is federated array (`ctx.auth.allowedSources`) > scalar (`ctx.sourceId`) > nothing. Don't
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak.
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak. Corollary
|
||||
(unscoped-check/scoped-write): `engine.getPage` with no opts matches ANY source while
|
||||
`putPage` defaults to `'default'` — an existence check + write pair must scope the read
|
||||
to the write's source (`getPage(slug, { sourceId: x ?? 'default' })`). Guarded by
|
||||
`scripts/check-getpage-scoped-write.mjs` (opt-out marker
|
||||
`gbrain-allow-unscoped-getpage` for read-only first-match sites).
|
||||
- **JSONB: never `JSON.stringify` into a `::jsonb` cast.** postgres.js double-encodes it (a jsonb
|
||||
string scalar); PGLite hides the bug. This bites BOTH spellings — the template form
|
||||
(`${JSON.stringify(x)}::jsonb`) AND the positional form (`executeRaw(\`…$N::jsonb\`, [JSON.stringify(x)])`,
|
||||
|
||||
@@ -1,5 +1,157 @@
|
||||
# TODOS
|
||||
|
||||
## CLI→MCP gap-closure wave follow-ups (2026-08-16; plan: ~/.claude/plans/system-instruction-you-are-working-concurrent-lantern.md)
|
||||
|
||||
- [ ] **P2 — publish-gate fail-open on a DB-config read failure.**
|
||||
**What:** `readPublishGate` + `assertPublishEnabled` (publish-gate path) fall back to the
|
||||
file plane when `engine.getConfig` throws, so a DB outage with file-plane=true but a
|
||||
DB-override=false widens authorization instead of denying it. **Why:** an auth gate that
|
||||
opens wider when its store is unreachable is fail-open — the wrong default for a
|
||||
publish/authorization boundary. **Context:** pre-existing behavior, explicitly pinned by
|
||||
`test/publish-gates.test.ts:71`; this needs a dedicated auth-plane decision (fail-closed
|
||||
vs. the current fail-open), NOT a drive-by flip in a test-regression pass. **Effort:** M,
|
||||
review-bound (one-way-door auth semantics).
|
||||
|
||||
- [ ] **P2 — takes-fence parser drops pack-extended kinds (whole-page refusal is the interim guard).**
|
||||
**What:** `parseTakesFence`'s `KIND_VALUES` is the closed `{fact,take,bet,hunch}` set, so a
|
||||
schema-pack kind (`finding|hypothesis|…`) is skipped as malformed and surfaces a warning —
|
||||
which is exactly why the F1 guard (`assertFenceRoundTrips`) has to refuse the WHOLE page to
|
||||
avoid deleting the skipped row on a re-render. **Why:** a brain with pack-extended takes
|
||||
kinds can't be mutated through the write verbs at all today (every mutate refuses
|
||||
`fence_unparsed`). **Deeper fix:** widen the parser to accept any string kind (`TakeKind`
|
||||
opened to `string` in v0.38) and/or make the fence editor splice-preserve raw unparsed
|
||||
lines instead of a whole-fence re-render. **Effort:** M. (Related to the P3 pack-aware
|
||||
kind-validation item below, but that one is write-side; this is the parser + editor.)
|
||||
|
||||
- [ ] **P2 — `get_health` migration-ledger honesty.**
|
||||
**What:** `loadCompletedMigrations` skips a malformed JSONL line with a `warn`, so a
|
||||
truncated ledger entry silently mis-reports — a completed migration can look pending —
|
||||
instead of surfacing a `ledger_unreadable` signal. **Why:** health/doctor output should
|
||||
fail loud when its own audit trail is unreadable, not quietly under-count. **Effort:** S.
|
||||
|
||||
- [ ] **P2 — `quarantine_list` SELECT projection pushdown.**
|
||||
**What:** the quarantine scan pulls full page bodies (`SELECT p.*`) only to read two
|
||||
frontmatter keys. Push the marker filter into SQL (`frontmatter ? 'quarantine'`) and
|
||||
project just `slug`, `source_id`, `frontmatter`. **Why:** loading every page body to check
|
||||
a frontmatter flag is O(corpus-bytes) for an O(matches) result. **Effort:** S.
|
||||
|
||||
- [ ] **P3 — `permissions.takes_write_holders`: split the takes read/write holder axes.**
|
||||
**What:** a dedicated write-side holder allow-list config, consumed by the takes write
|
||||
verbs' fence in `src/core/ops/takes.ts` (today the WRITE fence reuses the READ
|
||||
allow-list `takesHoldersAllowList` — fail-closed and symmetric, but semantically
|
||||
overloaded). **Why:** an operator may want an agent to READ private holders but WRITE
|
||||
only world-held rows, or vice versa. **Context:** decided at the wave's CEO/OV review
|
||||
("reuse now, split when field use demands"); the fence is one shared function
|
||||
(`takesWriteAllowList` + takes-write.ts's holder checks), so the split is a
|
||||
resolution-chain change, not a redesign. **Effort:** S. **Depends on:** field demand.
|
||||
|
||||
- [ ] **P3 — pack-aware takes kind validation, shared CLI + ops.**
|
||||
**What:** `takes_add`/`takes_supersede` pin `kind` to the 4 base literals
|
||||
(fact|take|bet|hunch) — same limitation as the CLI's `ensureKind` — while schema packs
|
||||
can extend `takes_kinds` (engine.ts TakeKindLiteral). Validate against the ACTIVE
|
||||
pack's kind set in ONE shared place (takes-write.ts) and widen the op enum note.
|
||||
**Why:** pack-extended kinds (finding|hypothesis|…) can't be written through either
|
||||
surface today. **Effort:** S.
|
||||
|
||||
- [ ] **P3 — `mcp:capture` provenance channel label (mini trust review).**
|
||||
**What:** capture delegates to put_page, so remote captures stamp `source_kind:
|
||||
'mcp:put_page'` (honest CV6 delegation; the op result carries channel:'capture').
|
||||
A distinct `mcp:capture` stamp needs a trusted internal channel label through the CV6
|
||||
else-branch — its own small trust review, filed rather than rushed. **Why:** finer
|
||||
provenance analytics on ingestion channels. **Effort:** S, review-bound.
|
||||
## Five-issue fix wave follow-ups (backlinks corruption / malformed paths / type warnings / getPage scoping / queue admission)
|
||||
|
||||
- [ ] **P2 — migrate the remaining fs writers to core/atomic-write.** **What:**
|
||||
`src/core/skillopt/apply-edits.ts` (atomicWrite, leaks tmp on write error),
|
||||
`src/core/write-through.ts` (own tmp+rename), `src/commands/lint.ts:~526`
|
||||
(bare writeFileSync in runLintCore) move onto `src/core/atomic-write.ts`
|
||||
(unique tmp + fsync + mode preservation + optional on-disk verify). Include
|
||||
page-lock unification: write-through's render does NOT take withPageLock, so
|
||||
the backlinks-vs-render lost-update race is only half-closed (backlinks
|
||||
locks; render doesn't). **Why:** four hand-rolled copies drift; the shared
|
||||
helper is strictly stronger. **Effort:** M. **Priority:** P2.
|
||||
- [ ] **P3 — relocate/retire skillopt's splitFrontmatter.** **What:** either
|
||||
move it to core/markdown.ts next to frontmatterBodyOffset or port its one
|
||||
SKILL.md caller onto the canonical helper (skillopt's regex is LF-at-byte-0
|
||||
only; the canonical one handles leading blanks + CRLF). **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — admission/stats indexes if hot.** **What:** expression index on
|
||||
`(name, (data->>'__param_hash')) WHERE status='waiting'` for the coalesce
|
||||
probe + `(name, created_at)` for the per-type stats aggregates, when
|
||||
minion_jobs exceeds ~100k rows. Same family as the buildQueueDepths perf
|
||||
note (status.ts) and the completed-recency probe TODO below. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P2 — getPage type-boundary redesign (the durable fix behind the
|
||||
guard).** **What:** make source scope explicit at the TYPE level — required
|
||||
scope param or an explicit ALL_SOURCES sentinel on `engine.getPage`, so an
|
||||
unscoped read is unrepresentable instead of merely linted
|
||||
(check-getpage-scoped-write.mjs is the interim guard; the default-first
|
||||
ORDER BY makes today's unscoped reads deterministic). ~78 call sites.
|
||||
**Effort:** L. **Priority:** P2.
|
||||
- [ ] **P2 — per-name claim fairness / lane isolation.** **What:** the
|
||||
admission wave (coalescing/TTL/quota) is deliberately submit-side only;
|
||||
claim order remains global FIFO per queue (`queue.ts` claim ORDER BY), so
|
||||
one divergent type still starves same-queue siblings until TTL/quota bites.
|
||||
A per-name claim budget or weighted claim is the drain-side primitive.
|
||||
**Effort:** L. **Priority:** P2.
|
||||
- [ ] **P3 — jobs stats divergence: per-queue scoping option.** **What:**
|
||||
the DIVERGENT scream computes name-global (matches quota semantics); a
|
||||
`--queue`-scoped variant would help multi-queue operators localize the
|
||||
producer. **Effort:** S. **Priority:** P3.
|
||||
- [ ] **P2 — requeue surface for waiting-TTL-cancelled jobs.** **What:**
|
||||
`jobs retry` targets failed/dead only; a TTL-cancelled row (error_text
|
||||
prefix `waiting_ttl_expired`) that turns out to have been wanted needs a
|
||||
`jobs requeue` (or a retry carve-out gated on that prefix) instead of
|
||||
hand-resubmitting. The data survives (cancelled rows keep payloads +
|
||||
free their idempotency keys), so this is purely a CLI surface. **Effort:**
|
||||
S. **Priority:** P2. (Pre-landing data-migration review, five-issue wave.)
|
||||
- [ ] **P2 — dream-path quota-degradation integration tests.** **What:**
|
||||
live-queue integration tests for the QueueQuotaExceededError consumers:
|
||||
cycle patterns → `skipped('admission_quota')`, synthesize → quota latch
|
||||
(one skip per remaining transcript, stop submitting), agent fanout →
|
||||
whole-tree cancel + exit 1. Unit seams exist (isQueueQuotaExceededError
|
||||
is pinned); what's missing is the end-to-end phase behavior under a
|
||||
1-quota config. **Effort:** M. **Priority:** P2.
|
||||
- [ ] **P3 — coalesce advisory-lock concurrency e2e.** **What:** real-PG
|
||||
e2e slamming N concurrent identical parentless submits → exactly one row
|
||||
(the advisory lock serializes (name, queue, hash)); PGLite can't prove
|
||||
this (single connection). Home: the DATABASE_URL-gated e2e lane.
|
||||
**Effort:** S. **Priority:** P3.
|
||||
- [ ] **P3 — consolidate the stable-stringify triplets.** **What:**
|
||||
`admission.ts` (param hash), plus the two earlier canonical-JSON copies
|
||||
(op-checkpoint hashing, cli-options) each roll their own sorted-key
|
||||
stringify; one `core/canonical-json.ts` would do. Hash-compat note: the
|
||||
admission copy feeds persisted `__param_hash` values — a behavior-change
|
||||
regression there just disables old-row coalescing (forward-safe), but
|
||||
keep the sorted-key semantics bit-identical anyway. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — reconcile lane: quarantine-not-delete option for malformed-path
|
||||
rows + doctor hint nuance.** **What:** full-sync reconcile hard-deletes
|
||||
poisoned rows (consistent with 'strategy' semantics); a
|
||||
`--quarantine-malformed` alternative would preserve rows for triage. Also
|
||||
the malformed_path_pages doctor hint could distinguish rows whose FILE
|
||||
still exists on disk (rename rescues content) from never-committed DB-only
|
||||
rows (delete is the only option). **Effort:** S. **Priority:** P3.
|
||||
- [ ] **P3 — thread source scope into `schema lint --with-db`.** **What:**
|
||||
the stored-type data-plane rules accept `LintOpts.sourceId` (multi-source
|
||||
brains can resolve different packs per source; comparing another source's
|
||||
rows against this manifest yields false alias/undeclared warnings), but
|
||||
neither `src/commands/schema.ts` (`runAllLintRules(pack, { engine })`) nor
|
||||
MCP `schema_lint` passes it — the CLI runs a global scan. Add
|
||||
`--source-id` / honor the worktree pin, and expose `[--json]` in the
|
||||
`jobs stats` usage line while in the area (`src/commands/jobs.ts:309`
|
||||
documents `--queue`/`--cluster-errors` but not the shipped `--json`).
|
||||
Also: the interactive coalesce hint suggests "pass a fresh idempotency
|
||||
key", which `gbrain agent run` has no flag for (raw `jobs submit` does).
|
||||
Surfaced by the v0.46.11.0 post-ship doc review. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — one-time cross-source clobber audit.** **What:** the
|
||||
pre-guard unscoped-check/scoped-write class could have historically
|
||||
written 'default'-source rows that shadow same-slug rows in other sources.
|
||||
A one-shot integrity probe (`SELECT slug FROM pages GROUP BY slug HAVING
|
||||
count(DISTINCT source_id) > 1` + updated_at ordering heuristics) would
|
||||
surface survivors for review. **Effort:** S. **Priority:** P3.
|
||||
|
||||
## Containment-sprint follow-ups (coverage truth + module peels; plan: ~/.claude/plans/system-instruction-you-are-working-serialized-forest.md)
|
||||
|
||||
- [ ] **P1 — Graduate the diff-coverage gate to blocking (time-boxed 2 weeks from merge).**
|
||||
@@ -127,12 +279,19 @@ Staged-deletion discipline (ship replacements → migrate call sites → update
|
||||
registry entries (recipes/index.ts); `zeroEntropyCompatFetch`,
|
||||
`MAX_ZEROENTROPY_RESPONSE_BYTES`, `ZeroEntropyResponseTooLargeError` + the
|
||||
fetch-ternary arm (gateway.ts); ZE sets in dims.ts; `ze-switch.ts` +
|
||||
`retrieval-upgrade-planner.ts` + `retrieval-upgrade-prompt.ts` (~1200 lines) +
|
||||
cli.ts dispatch/CLI_ONLY/flag-registry rows; `checkZeEmbeddingHealth` in doctor
|
||||
`retrieval-upgrade-planner.ts` + cli.ts dispatch/CLI_ONLY/CLI_ONLY_SELF_HELP/
|
||||
SELF_HELP_WITHOUT_ENGINE/flag-registry rows; `checkZeEmbeddingHealth` in doctor
|
||||
(`provider_sunset` STAYS and goes generic — read `recipe.sunset` instead of the
|
||||
hardcoded ZE constants); pricing rows LAST (budget-tracker rerank metering reads
|
||||
them for historical audit rows). NOTE: test/ai/zeroentropy-compat-fetch.test.ts
|
||||
greps gateway.ts SOURCE TEXT — delete the test with the code, in the same commit.
|
||||
ALREADY DONE by the interim ZE cleanup wave (pre-Sept): `retrieval-upgrade-prompt.ts`
|
||||
deleted (banner/marketing copy gone); `ze-switch.ts` is now a ~170-line pure
|
||||
refusal/redirect shim (undo/dry-run ACTIONS retired — apply/undo wrote DB-plane
|
||||
config the file-plane-canonical runtime never read); `providers env`/`explain` are
|
||||
sunset-aware via the shared `sunsetMarker` in providers.ts (generic on
|
||||
`recipe.sunset` — the removal wave inherits it); `ze_embedding_health`'s missing-key
|
||||
copy is migration-first (the check itself still gets deleted here).
|
||||
- [ ] **P1 — Self-host continuity decision.** The v0.46.3 playbook's zero-re-embed
|
||||
path keeps the `zeroentropyai:zembed-1` id behind a base-URL override to a
|
||||
ZE-wire-compatible endpoint. Recipe deletion breaks it. Decide: keep a minimal
|
||||
@@ -142,11 +301,14 @@ Staged-deletion discipline (ship replacements → migrate call sites → update
|
||||
playbook (skills/migrations/v0.46.3.0.md) links here — honor it.
|
||||
- [ ] **P2 — Tests + CI.** Delete the 8 ZE-dedicated test files
|
||||
(zeroentropy-recipe, zeroentropy-compat-fetch, dims-zeroentropy,
|
||||
e2e/zeroentropy-live, ze-switch-cli, ze-switch-env-override, doctor-ze-checks,
|
||||
provider-sunset-doctor.serial gets REWRITTEN generic not deleted) + update ~40
|
||||
coupled files; drop the zeroentropy-live job + ZEROENTROPY_API_KEY secret from
|
||||
.github/workflows/e2e.yml:168,179 (already date-skip-gated since v0.46.3);
|
||||
scripts/test-weights.json rows.
|
||||
e2e/zeroentropy-live, ze-switch-cli [now pins the shim contract — dies with the
|
||||
shim], ze-switch-env-override [pins the planner's test-only functions],
|
||||
doctor-ze-checks, provider-sunset-doctor.serial gets REWRITTEN generic not
|
||||
deleted) + update ~40 coupled files; drop the zeroentropy-live job +
|
||||
ZEROENTROPY_API_KEY secret from .github/workflows/e2e.yml:239,250,377 (line refs
|
||||
refreshed by the interim cleanup wave; already date-skip-gated since v0.46.3);
|
||||
scripts/test-weights.json rows. Also remove 'ze-switch' from the
|
||||
cli-help-without-brain HELP_WITHOUT_BRAIN list when the shim dies.
|
||||
- [ ] **P2 — Config + docs.** `zeroentropy_api_key` config key: keep
|
||||
parseable-but-warned (removing it would make old config.json files fail to
|
||||
load); delete docs/ai-providers/zeroentropy.md + its scripts/llms-config.ts
|
||||
@@ -381,10 +543,15 @@ Each was explicitly deferred in the pass's CEO/eng/outside-voice reviews.
|
||||
- [ ] **P2 — `jobs submit --max-pending` public flag.** maxPending stays an
|
||||
internal submit option this wave (Codex C4): its semantics exclude
|
||||
delayed/paused/waiting-children rows, and identity is (name, queue, source)
|
||||
so distinct payloads collapse. Decide the public contract (include delayed?
|
||||
explicit scope key?) after the primitive soaks in autopilot, then mirror
|
||||
parseMaxWaitingFlag (clamp [1,100]) + help + flag-registry regen + optional
|
||||
submit_job MCP param. Where: src/commands/jobs.ts, src/core/operations.ts.
|
||||
so distinct payloads collapse. NOTE (five-issue fix wave): the
|
||||
payload-DISTINCT dedupe primitive now exists — admission param-coalescing
|
||||
(`coalesce_params` / minions.coalesce_params.<name>, hash of the full
|
||||
payload incl. owner lane) covers the "identical submits collapse, distinct
|
||||
ones don't" case; --max-pending remains the single-flight-per-scope story.
|
||||
Decide the public contract (include delayed? explicit scope key?) after the
|
||||
primitive soaks in autopilot, then mirror parseMaxWaitingFlag (clamp
|
||||
[1,100]) + help + flag-registry regen + optional submit_job MCP param.
|
||||
Where: src/commands/jobs.ts, src/core/operations.ts.
|
||||
- [ ] **P2 — maxPending at the other single-flight dispatch sites.** The
|
||||
freshness sync submit (src/commands/autopilot.ts freshness loop) and the
|
||||
targeted remediation steps (autopilot.ts targeted-submit loop) still use
|
||||
@@ -4136,6 +4303,16 @@ verify Voyage adapter integration in `src/core/ai/recipes/voyage.ts`).
|
||||
### Token rotation: `gbrain auth rotate <name>` + `rotate_token` MCP op
|
||||
**Priority:** P2
|
||||
|
||||
**Deferral note (CLI→MCP gap-closure wave, 2026-08-16, user decision D3A):**
|
||||
deliberately NOT bundled into the gap-closure wave — there is no CLI to
|
||||
mirror yet (this TODO is its own work item, not a CLI→MCP gap), and a token
|
||||
that can mint its own successor turns a leaked credential into persistence +
|
||||
operator lock-out, so it needs its own auth-plane design pass. Sketch agreed
|
||||
at review: admin scope, NOT localOnly (remote rotation is the point),
|
||||
SELF-rotation only (the calling token/client — never a name param), returns
|
||||
the new secret exactly once, rate-limited via the RateLimiter house pattern,
|
||||
and ships in the same PR as the `gbrain auth rotate` CLI.
|
||||
|
||||
**What:** Atomic rotate for legacy + OAuth tokens. Issue a new token in the same TX as the revocation of the old, no overlap window. Refresh-token rotation already exists for OAuth; this is the unified user-facing surface (CLI + MCP).
|
||||
|
||||
**Why:** Today rotation is `revoke + create`, with a window where neither token works. For long-lived bearer keys handed to agents, that's a reload outage every time the key gets rotated.
|
||||
@@ -4146,7 +4323,15 @@ verify Voyage adapter integration in `src/core/ai/recipes/voyage.ts`).
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Migration introspection in `get_health`
|
||||
**Priority:** P3
|
||||
**Priority:** P3 — **DONE (CLI→MCP gap-closure wave, 2026-08-16).** The
|
||||
`get_health` OP now returns `migrations {pending, partial, wedged,
|
||||
skipped_future}` composed at the op layer from the new
|
||||
`src/core/migration-ledger.ts` (version strings only). Op-layer composition
|
||||
was chosen over this TODO's engine-method wording: the ledger is a
|
||||
filesystem JSONL, engine-agnostic — growing `BrainEngine.getHealth()` would
|
||||
have duplicated a file read in both engines. Pinned by
|
||||
`test/migration-ledger.test.ts` + `test/get-job-stats-op.test.ts`'s sibling
|
||||
patterns.
|
||||
|
||||
**What:** Extend `BrainEngine.getHealth()` return shape with `migrations: { pending: [...], wedged: [...] }`. `gbrain doctor` already shows this; expose it via the MCP op so remote agents can detect partial-migration state without invoking `doctor` separately.
|
||||
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ claude mcp add gbrain -- gbrain serve --surface verbs # Claude Code
|
||||
codex mcp add gbrain -- gbrain serve --surface verbs # Codex
|
||||
```
|
||||
|
||||
The agent spawns `gbrain serve` as a stdio subprocess against your local brain. `--surface verbs` gives the agent the seven-verb memory protocol (`recall`, `remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` — [MEMORY_VERBS v1](protocol/MEMORY_VERBS_v1.md)) instead of the full tool catalog; `--surface starter` adds the daily-driver set on top of the verbs (~26 ops total); drop the flag (default `full`) for every operation. Full walkthrough (both this local path and connecting to a remote brain), plus the brain-first protocol to paste into `CLAUDE.md` / `AGENTS.md`: **[Give your coding agent a memory](tutorials/connect-coding-agent.md)**.
|
||||
The agent spawns `gbrain serve` as a stdio subprocess against your local brain. `--surface verbs` gives the agent the seven-verb memory protocol (`recall`, `remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` — [MEMORY_VERBS v1](protocol/MEMORY_VERBS_v1.md)) instead of the full tool catalog; `--surface starter` adds the daily-driver set on top of the verbs (~27 ops total); drop the flag (default `full`) for every operation. Full walkthrough (both this local path and connecting to a remote brain), plus the brain-first protocol to paste into `CLAUDE.md` / `AGENTS.md`: **[Give your coding agent a memory](tutorials/connect-coding-agent.md)**.
|
||||
|
||||
## 3. MCP server (any MCP client)
|
||||
|
||||
|
||||
+13
-2
@@ -4,15 +4,16 @@
|
||||
<!-- Regenerate: bun run scripts/generate-tool-catalog.ts -->
|
||||
<!-- Freshness-guarded by scripts/check-tool-catalog-fresh.sh (bun run verify). -->
|
||||
|
||||
Every non-localOnly operation on the MCP surface: 104 tools across 22 areas. **Starter** marks membership in the ~26-op `starter` surface (`src/mcp/surface.ts`); **Gate** names the config key that must be true before remote callers see/call the op (`gbrain config set <key> true`). What a given token actually sees is further filtered per request by scope, bound-client fence, publish gates, and the per-client surface — see `docs/operations/mcp-surface-runbook.md`. Area names are non-contractual groupings.
|
||||
Every non-localOnly operation on the MCP surface: 115 tools across 22 areas. **Starter** marks membership in the ~27-op `starter` surface (`src/mcp/surface.ts`); **Gate** names the config key that must be true before remote callers see/call the op (`gbrain config set <key> true`). What a given token actually sees is further filtered per request by scope, bound-client fence, publish gates, and the per-client surface — see `docs/operations/mcp-surface-runbook.md`. Area names are non-contractual groupings.
|
||||
|
||||
## admin
|
||||
|
||||
| Tool | Description | Scope | Starter | Gate |
|
||||
|---|---|---|---|---|
|
||||
| `get_health` | Brain health dashboard (embed coverage, stale pages, orphans) | admin | | |
|
||||
| `get_health` | Brain health dashboard (embed coverage, stale pages, orphans). | admin | | |
|
||||
| `get_stats` | Brain statistics (page count, chunk count, etc.) | admin | | |
|
||||
| `get_status_snapshot` | Snapshot for `gbrain status` thin-client mode: sync freshness + last cycle + queue depths + worker liveness. | admin | | |
|
||||
| `quarantine_list` | List quarantined (hidden) and optionally content-flagged pages by scanning page frontmatter, newest-updated first. | admin | | |
|
||||
| `run_doctor` | Run brain health checks and return a structured DoctorReport (thin-client doctor surface). | admin | | |
|
||||
| `run_onboard` | Probe brain health + optionally submit onboard remediations. | admin | | |
|
||||
| `run_skillopt` | Run SkillOpt against a single skill. | admin | | |
|
||||
@@ -91,6 +92,7 @@ Every non-localOnly operation on the MCP surface: 104 tools across 22 areas. **S
|
||||
| `get_agent_job` | Poll an agent job submitted via submit_agent. | agent | yes | |
|
||||
| `get_job` | Get job status and details by ID | admin | | |
|
||||
| `get_job_progress` | Get structured progress for a running job | admin | | |
|
||||
| `get_job_stats` | Job queue statistics. | admin | | |
|
||||
| `list_jobs` | List jobs with optional filters | admin | | |
|
||||
| `pause_job` | Pause a waiting, active, or delayed job | admin | | |
|
||||
| `replay_job` | Replay a completed/failed/dead job, optionally with modified data | admin | | |
|
||||
@@ -144,6 +146,7 @@ Every non-localOnly operation on the MCP surface: 104 tools across 22 areas. **S
|
||||
|
||||
| Tool | Description | Scope | Starter | Gate |
|
||||
|---|---|---|---|---|
|
||||
| `capture` | Capture a quick note into the brain — the "just remember this" write. | write | yes | |
|
||||
| `delete_page` | Soft-delete a page. | write | | |
|
||||
| `get_chunks` | Get content chunks for a page | read | | |
|
||||
| `get_page` | Read a page by slug (supports optional fuzzy matching). | read | yes | |
|
||||
@@ -174,9 +177,13 @@ Every non-localOnly operation on the MCP surface: 104 tools across 22 areas. **S
|
||||
|
||||
| Tool | Description | Scope | Starter | Gate |
|
||||
|---|---|---|---|---|
|
||||
| `cache_stats` | Semantic query-cache introspection: resolved knobs (enabled, similarity threshold, TTL) plus row counts and total hits. | admin | | |
|
||||
| `query` | Hybrid search with vector + keyword + multi-query expansion. | read | yes | |
|
||||
| `search` | Cheap hybrid search (vector + keyword + RRF) with no LLM expansion. | read | yes | |
|
||||
| `search_by_image` | v0.36 cross-modal Phase 2: image-as-query retrieval. | read | | |
|
||||
| `search_modes` | Read-only search-mode dashboard: active mode, per-knob resolved value with attribution (mode default vs config override), and the three frozen bundles. | read | | |
|
||||
| `search_stats` | Search observability over a window: cache hit rate, intent/mode mix, budget drops, rank-1 score drift, graph-signals failure counts. | admin | | |
|
||||
| `search_tune` | Read-only tuning recommendations derived from the last 7 days of search telemetry: what should change, why, and the paste-ready config command per recommendation — relay them to the user. | admin | | |
|
||||
|
||||
## skills
|
||||
|
||||
@@ -207,10 +214,14 @@ Every non-localOnly operation on the MCP surface: 104 tools across 22 areas. **S
|
||||
|
||||
| Tool | Description | Scope | Starter | Gate |
|
||||
|---|---|---|---|---|
|
||||
| `takes_add` | Record a take (typed claim) on a page: fact / take / bet / hunch, with a holder (who holds the belief: world, people/<slug>, companies/<slug>, or brain), weight 0..1, and optional source/since date. | write | | |
|
||||
| `takes_calibration` | Calibration curve: resolved correct/incorrect bets binned by stated weight; observed vs predicted per bucket. | read | | |
|
||||
| `takes_list` | List takes (typed/weighted/attributed claims) filtered by holder/kind/active/etc. | read | | |
|
||||
| `takes_resolve` | Resolve a take: quality correct / incorrect / partial / unresolvable, with optional evidence text and measured value/unit. | write | | |
|
||||
| `takes_scorecard` | Calibration scorecard for resolved bets: counts, accuracy, Brier (correct ∨ incorrect only), partial_rate. | read | | |
|
||||
| `takes_search` | Keyword search across takes (pg_trgm similarity over claim text) | read | | |
|
||||
| `takes_supersede` | Supersede a take with a replacement claim: the old row is struck through (kept for archaeology), the replacement appends at the next fence row number. | write | | |
|
||||
| `takes_update` | Update a take's mutable fields (weight, source, since date). | write | | |
|
||||
| `think` | Multi-hop synthesis across pages + takes + graph. | read | | |
|
||||
|
||||
## timeline
|
||||
|
||||
@@ -6,10 +6,13 @@
|
||||
> recipe: `gbrain init` auto-pick and the interactive picker exclude it
|
||||
> (explicit `--embedding-model zeroentropyai:*` still works, with a loud
|
||||
> warning), every ZE embed/rerank call prints a once-per-process
|
||||
> deprecation warning, `gbrain providers` annotates it DEPRECATED, and
|
||||
> `gbrain ze-switch` refuses to switch a brain ONTO ZeroEntropy (`--undo`
|
||||
> and `--dry-run` still work). The September release removes the recipe
|
||||
> entirely. A brain still embedding through the hosted API loses semantic
|
||||
> deprecation warning, `gbrain providers` annotates it DEPRECATED
|
||||
> (`gbrain providers env zeroentropyai` prints this off-ramp instead of a
|
||||
> signup link), and
|
||||
> `gbrain ze-switch` is a pure refusal/redirect shim (every invocation
|
||||
> refuses or redirects; `--undo` prints the exact migrate command that
|
||||
> returns a switched brain to its prior provider — it no longer acts).
|
||||
> The September release removes the recipe entirely. A brain still embedding through the hosted API loses semantic
|
||||
> retrieval entirely on the shutdown date: query embedding uses the same
|
||||
> endpoint, so **existing vectors become unqueryable**, not just new
|
||||
> content. Two fixes, either works:
|
||||
@@ -42,28 +45,29 @@
|
||||
>
|
||||
> The hosted setup below remains accurate until the shutdown date.
|
||||
|
||||
[ZeroEntropy](https://zeroentropy.dev) ships two specialized small models
|
||||
for retrieval pipelines:
|
||||
[ZeroEntropy](https://zeroentropy.dev) shipped two specialized small
|
||||
models for retrieval pipelines (factual specs kept for existing users and
|
||||
self-hosters — this is not a recommendation):
|
||||
|
||||
- **`zembed-1`** — multilingual embedding distilled from zerank-2.
|
||||
Flexible Matryoshka dims (2560/1280/640/320/160/80/40), 32K context,
|
||||
asymmetric `input_type: query|document` encoding. $0.025/1M tokens
|
||||
(sale) / $0.05 regular.
|
||||
- **`zerank-2`** — SOTA multilingual cross-encoder reranker.
|
||||
$0.025/1M tokens (~50% cheaper than Cohere/Voyage rerankers).
|
||||
Plus `zerank-1` and `zerank-1-small` for legacy / open-source needs.
|
||||
asymmetric `input_type: query|document` encoding.
|
||||
- **`zerank-2`** — multilingual cross-encoder reranker. Plus `zerank-1`
|
||||
and `zerank-1-small` (open-source weights).
|
||||
|
||||
Both land in gbrain v0.35.0.0 behind the openai-compatible recipe path,
|
||||
Both landed in gbrain v0.35.0.0 behind the openai-compatible recipe path,
|
||||
alongside OpenAI and Voyage.
|
||||
|
||||
## Setup
|
||||
## Setup (existing brains and self-hosters only — do not onboard)
|
||||
|
||||
1. Get an API key at
|
||||
[dashboard.zeroentropy.dev](https://dashboard.zeroentropy.dev).
|
||||
2. Export it:
|
||||
```bash
|
||||
export ZEROENTROPY_API_KEY=<your-key>
|
||||
```
|
||||
New installs use Voyage (`gbrain init` handles it); do not create a new
|
||||
ZeroEntropy account for a provider that shuts down on 2026-09-04. A brain
|
||||
that already has a key exports it as before for the remaining hosted
|
||||
window:
|
||||
|
||||
```bash
|
||||
export ZEROENTROPY_API_KEY=<your-existing-key>
|
||||
```
|
||||
|
||||
## Leaving ZeroEntropy (the off-ramp)
|
||||
|
||||
@@ -124,19 +128,12 @@ the key, every rerank call fails-open (audit-logged) and search returns
|
||||
RRF order — same UX as before, just with an observable failure surfaced
|
||||
via `gbrain doctor`.
|
||||
|
||||
### Opt-in on `conservative` mode
|
||||
### Enabling reranking today
|
||||
|
||||
```bash
|
||||
gbrain config set search.reranker.enabled true
|
||||
```
|
||||
|
||||
The override sits above the mode-bundle default; opt-out is one flip.
|
||||
|
||||
### Cost anchor
|
||||
|
||||
At 30 candidates × ~400 tokens/chunk × $0.025/1M = **~$0.0003/query**.
|
||||
Rounding error against the `tokenmax + Opus` pairing's ~$700/mo at
|
||||
single-user volume per the CLAUDE.md cost matrix.
|
||||
Set the surviving reranker FIRST, then enable — enabling on a brain that
|
||||
never set `search.reranker.model` falls back to the dying `zerank-2`:
|
||||
`gbrain config set search.reranker.model voyage:rerank-2.5`, then
|
||||
`gbrain config set search.reranker.enabled true`.
|
||||
|
||||
### Verify
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -69,7 +69,7 @@ gbrain schema fork <a> <b> # copy + rename a pack (experimental)
|
||||
gbrain schema edit <name> # surface the pack path (experimental)
|
||||
gbrain schema diff <a> <b> # set-diff two packs (experimental)
|
||||
gbrain schema graph # ASCII type listing (experimental)
|
||||
gbrain schema lint # flag duplicates + missing prefixes
|
||||
gbrain schema lint [--with-db] # duplicates + missing prefixes; --with-db adds data-plane rules
|
||||
gbrain schema explain <type> # plain-English type description (experimental)
|
||||
gbrain schema downgrade --to <p> # restore previous pack (recovery)
|
||||
gbrain schema usage --since 30d # per-verb invocation counts (telemetry)
|
||||
@@ -79,6 +79,18 @@ The verbs marked `experimental` are demand-gated: usage is tracked via the
|
||||
schema-events audit (`gbrain schema usage`), which informs whether
|
||||
rarely-used verbs get deprecated.
|
||||
|
||||
With `--with-db`, `schema lint` also runs two data-plane rules over the
|
||||
stored corpus: `stored_type_is_alias` (a page's explicit type is an alias —
|
||||
the canonical type and its filing directory are named) and
|
||||
`stored_type_undeclared` (the type isn't in the active pack at all). The
|
||||
rule layer accepts a per-source scope (`LintOpts.sourceId` — multi-source
|
||||
brains can resolve different packs per source), though the CLI currently
|
||||
runs a global scan. The same classification warns once per type per run at
|
||||
sync/import so alias types stop filing into unexpected directories
|
||||
silently; silence the ingest warnings with
|
||||
`gbrain config set schema.type_warnings false` (the `--with-db` lint rules
|
||||
are unaffected).
|
||||
|
||||
## Resolution chain (7 tiers)
|
||||
|
||||
When the engine decides "which pack is active for this query?", it walks
|
||||
|
||||
@@ -221,15 +221,19 @@ Wire the harness to drive 3 embedding providers via the newly-exposed gbrain gat
|
||||
```
|
||||
|
||||
### Smoke verification (run manually before opening PR)
|
||||
|
||||
> (Historical: the two `zeroentropyai:` commands below stop passing after
|
||||
> 2026-09-04 — do not run them. Only the non-ZE smokes remain runnable.)
|
||||
|
||||
```bash
|
||||
bun run eval:smoke -- --embedder openai:text-embedding-3-large --dim 1536
|
||||
bun run eval:smoke -- --embedder voyage:voyage-4-large --dim 2048
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560 --reranker zeroentropyai:zerank-2
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560 # historical
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560 --reranker zeroentropyai:zerank-2 # historical
|
||||
```
|
||||
|
||||
All four MUST exit 0. Reports should print the observed vector dim, matching the
|
||||
configured dim.
|
||||
The two non-ZE smokes MUST exit 0 (the ZE pair did at the time). Reports
|
||||
should print the observed vector dim, matching the configured dim.
|
||||
|
||||
### Open PR β
|
||||
```bash
|
||||
|
||||
@@ -123,8 +123,8 @@ ingestion — not just new content.
|
||||
census, the un-merged file plane), so a pre-set env var cannot fake a
|
||||
completed migration.
|
||||
5. **Apply.** When the target width differs from the actual column width,
|
||||
runs the same atomic schema transition `ze-switch` uses, in one
|
||||
transaction. It rebuilds **all three dim-pinned text-embedding-space
|
||||
runs the atomic schema transition owned by `embedding-migration.ts`
|
||||
(the survivor module), in one transaction. It rebuilds **all three dim-pinned text-embedding-space
|
||||
columns** — `content_chunks.embedding`, `query_cache.embedding`, and
|
||||
`facts.embedding` — at the new width, preserving each column's type
|
||||
(`vector` vs `halfvec`) and recreating its HNSW index. Missing any of the
|
||||
|
||||
@@ -12,9 +12,13 @@ The persistent worker can die silently from:
|
||||
- Bun process crashes with no automatic restart.
|
||||
- Internal event-loop death (PID alive, worker loop stopped).
|
||||
|
||||
When the worker dies, submitted jobs sit in `waiting` forever. The
|
||||
canonical answer is `gbrain jobs supervisor` — a first-class CLI that
|
||||
spawns `gbrain jobs work` as a child and auto-restarts it on crash.
|
||||
When the worker dies, submitted jobs sit in `waiting` — indefinitely for
|
||||
most types; types with a waiting-TTL (`subagent` defaults to 48h, see the
|
||||
[queue operations runbook](queue-operations-runbook.md)) are eventually
|
||||
cancelled with an auditable reason rather than queueing forever. Either
|
||||
way the work doesn't happen. The canonical answer is
|
||||
`gbrain jobs supervisor` — a first-class CLI that spawns `gbrain jobs work`
|
||||
as a child and auto-restarts it on crash.
|
||||
|
||||
## Worker supervision
|
||||
|
||||
|
||||
@@ -55,6 +55,45 @@ gbrain jobs supervisor stop && gbrain jobs supervisor start --detach --json
|
||||
gbrain jobs retry <id>
|
||||
```
|
||||
|
||||
## The backlog grows structurally (DIVERGENT QUEUE)
|
||||
|
||||
A different failure from a wedge: the worker is draining fine, but one job
|
||||
type's intake structurally exceeds its completions, so the waiting pile
|
||||
grows forever. Since v0.46.11.0 the queue has admission control and the
|
||||
signal is loud:
|
||||
|
||||
```bash
|
||||
gbrain jobs stats # Drained/Waiting columns + a DIVERGENT QUEUE
|
||||
# scream per offending type (also in --json)
|
||||
gbrain doctor --json | jq '.checks[] | select(.name == "queue_health")'
|
||||
# same findings for cron topologies
|
||||
```
|
||||
|
||||
The scream fires when a type's 24h intake exceeds `GBRAIN_QUEUE_DIVERGENCE_RATIO`
|
||||
(default 2) × its 24h completions AND more than
|
||||
`GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING` (default 50) jobs are waiting.
|
||||
Cancellations — including the waiting-TTL sweep — are deliberately not
|
||||
counted as drain: outflow is not work.
|
||||
|
||||
What's already protecting you, and the knobs:
|
||||
|
||||
- **Param-coalescing** (default on for `subagent`): identical parentless
|
||||
submits — same owner lane, payload, and execution options — coalesce onto
|
||||
the existing waiting job instead of stacking. Per-name toggle:
|
||||
`minions.coalesce_params.<name>`.
|
||||
- **Waiting-TTL** (default 48h for `subagent`): jobs still waiting past the
|
||||
TTL are cancelled with an auditable reason instead of queueing forever.
|
||||
Tune or disable: `gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`.
|
||||
The first sweep never fires cold — a one-time notice prints with the
|
||||
affected-job count, then a one-hour grace window holds before the first
|
||||
cancellation.
|
||||
- **Waiting quota** (opt-in, off by default): a hard cap on a type's waiting
|
||||
count, name-global across queues, exact under concurrent submitters. New
|
||||
submits past the cap are rejected with a structured, retryable error.
|
||||
Opt in: `gbrain config set minions.quota_max_waiting.<name> <n>`.
|
||||
- **Kill-switch**: `GBRAIN_MINIONS_ADMISSION=0` disables all three at once
|
||||
(incident escape hatch, no DB needed).
|
||||
|
||||
## Triage commands
|
||||
|
||||
```bash
|
||||
@@ -106,6 +145,16 @@ gbrain jobs smoke --wedge-rescue
|
||||
drain them. Set `--max-waiting N` on the submission or on the programmatic
|
||||
`queue.add()` call. If you want a taller pile, raise the threshold via
|
||||
`GBRAIN_QUEUE_WAITING_THRESHOLD=50 gbrain doctor`.
|
||||
- **divergent queue** — A type's 24h intake structurally exceeds its 24h
|
||||
completions while a real backlog waits (same thresholds as the
|
||||
`jobs stats` scream, so the two surfaces agree). The finding names the
|
||||
type and prints the exact `minions.quota_max_waiting.<name>` command to
|
||||
cap admission. See "The backlog grows structurally" above.
|
||||
- **waiting-TTL cancellations** — The admission sweep cancelled queued work
|
||||
that expired unclaimed in the last 24h. That's operating as designed, but
|
||||
it means the divergence is being shredded, not worked — intake still
|
||||
exceeds drain. Tune with `gbrain config set
|
||||
minions.ttl_waiting_hours.<name> <hours|0>`.
|
||||
|
||||
## Lock-renewal: reading an eviction, and the knobs
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ The resolved provider + dimensions get persisted to `~/.gbrain/config.json` atom
|
||||
|
||||
**Note on local providers.** Ollama and llama-server have no required API key, so they don't show up in env-detection auto-pick. Pick them explicitly with `--embedding-model ollama:<model>` to avoid silently routing to a daemon that may not be running.
|
||||
|
||||
**Note on the ZeroEntropy hosted API.** ZeroEntropy announced (2026-07-24) that its hosted endpoints shut down on **2026-09-04**, and the recipe is deprecated: init auto-pick and the interactive picker exclude it (explicit `--embedding-model zeroentropyai:*` still works, with a loud warning), every ZE embed/rerank call prints a once-per-process deprecation warning, and `gbrain providers` annotates it DEPRECATED. A brain still embedding through the hosted API loses semantic retrieval entirely on that date — query embedding uses the same endpoint, so existing vectors become unqueryable, not just new content. The off-ramp: `gbrain migrate embeddings --to voyage:voyage-4 --dim 1024 --dry-run` (cost preview), then `--yes`. 1280 is not a valid Voyage width (valid: 256/512/1024/2048), so a 1280d brain gets a one-time schema/HNSW rebuild to 1024; the OpenAI alternative keeps the width (flexible dims): `--to openai:text-embedding-3-small --dim 1280`. See [the migration guide](../guides/embedding-migration.md). Self-hosting the Apache-2.0 zembed-1 weights keeps every existing vector with zero re-embed, but the endpoint must speak ZeroEntropy's wire dialect — a generic OpenAI-compatible llama-server/Ollama will NOT work without a compat proxy (details in [`docs/ai-providers/zeroentropy.md`](../ai-providers/zeroentropy.md)). `gbrain doctor` (check `provider_sunset`) flags affected brains — including ZE-backed custom embedding columns — and prints target-aware paste-ready commands (Voyage at 1024; OpenAI keep-width when the brain's actual width is valid there); accepted the risk? `gbrain config set doctor.suppress_provider_sunset true` silences it.
|
||||
**Note on the ZeroEntropy hosted API.** ZeroEntropy announced (2026-07-24) that its hosted endpoints shut down on **2026-09-04**, and the recipe is deprecated: init auto-pick and the interactive picker exclude it (explicit `--embedding-model zeroentropyai:*` still works, with a loud warning), every ZE embed/rerank call prints a once-per-process deprecation warning, and `gbrain providers` annotates it DEPRECATED (`providers env zeroentropyai` prints the deprecation notice + migration command instead of the signup funnel, `providers explain` leads the row with ⚠ regardless of key readiness, and `gbrain doctor`'s ZE missing-key hint is migration-first). A brain still embedding through the hosted API loses semantic retrieval entirely on that date — query embedding uses the same endpoint, so existing vectors become unqueryable, not just new content. The off-ramp: `gbrain migrate embeddings --to voyage:voyage-4 --dim 1024 --dry-run` (cost preview), then `--yes`. 1280 is not a valid Voyage width (valid: 256/512/1024/2048), so a 1280d brain gets a one-time schema/HNSW rebuild to 1024; the OpenAI alternative keeps the width (flexible dims): `--to openai:text-embedding-3-small --dim 1280`. See [the migration guide](../guides/embedding-migration.md). Self-hosting the Apache-2.0 zembed-1 weights keeps every existing vector with zero re-embed, but the endpoint must speak ZeroEntropy's wire dialect — a generic OpenAI-compatible llama-server/Ollama will NOT work without a compat proxy (details in [`docs/ai-providers/zeroentropy.md`](../ai-providers/zeroentropy.md)). `gbrain doctor` (check `provider_sunset`) flags affected brains — including ZE-backed custom embedding columns — and prints target-aware paste-ready commands (Voyage at 1024; OpenAI keep-width when the brain's actual width is valid there); accepted the risk? `gbrain config set doctor.suppress_provider_sunset true` silences it.
|
||||
|
||||
## If first import fails
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ tunnel, no token needed. Works with both PGLite and Supabase engines.
|
||||
`entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)),
|
||||
the surface built for agents and quickstarts. `--surface starter` adds the
|
||||
daily-driver set on top (~26 ops total). Drop the flag for the full
|
||||
daily-driver set on top (core page/search/graph ops + capture). Drop the flag for the full
|
||||
operation catalog (`get_page`, `put_page`, `search`, graph ops, …) — `full` is
|
||||
the default and what existing installs already run.
|
||||
|
||||
@@ -118,8 +118,10 @@ You should see results from your GBrain knowledge base.
|
||||
> `gbrain config set mcp.publish_skills true`. Skill discovery and the core tools
|
||||
> named here (search, query, get_page, put_page, think, find_experts) are
|
||||
> full-surface — on `--surface verbs` the agent sees only the seven memory verbs,
|
||||
> and `list_skills` isn't on the surface at all. Note: `capture` is a
|
||||
> CLI-only command, not an MCP tool — the agent writes over MCP with `put_page`.
|
||||
> and `list_skills` isn't on the surface at all. `capture` is on the starter and
|
||||
> full surfaces (prefer it for quick notes — auto-slug + dedupe; `put_page` for
|
||||
> full-control writes); if your tool list doesn't carry it, use `put_page`, or
|
||||
> `remember` on the verbs surface.
|
||||
> Why brains differ on the default: [tutorial A1](../tutorials/connect-coding-agent.md#a1-on-the-host-serve-over-http).
|
||||
|
||||
## Ambient recall at session boundaries (v0.45.7)
|
||||
|
||||
+5
-3
@@ -39,7 +39,7 @@ that exact install one-liner on stderr; with no brain, it exits with
|
||||
resolution order: `$GBRAIN_BIN` → `~/.bun/bin/gbrain` → `gbrain` on PATH — the
|
||||
sanctioned install location is preferred over PATH so a stray `gbrain` earlier
|
||||
on PATH can't shadow it).
|
||||
`starter` is the 26-op daily-driver surface (the seven memory verbs + daily
|
||||
`starter` is the daily-driver surface (the seven memory verbs + daily
|
||||
brain ops) — the curated skills drive everything else through the `gbrain`
|
||||
CLI. Widen a machine without editing the snapshot: `GBRAIN_SURFACE=full` in
|
||||
the env that launches Codex (new sessions pick it up), or use the bootstrap
|
||||
@@ -129,8 +129,10 @@ everything it can do.
|
||||
|
||||
> **`list_skills` empty?** It's gated by `mcp.publish_skills` on the host — enable
|
||||
> it with `gbrain config set mcp.publish_skills true`. The core tools (search,
|
||||
> query, get_page, put_page, think, find_experts) work regardless; `capture` is
|
||||
> CLI-only, so write over MCP with `put_page`. Why brains differ on the default:
|
||||
> query, get_page, put_page, capture, think, find_experts) work regardless —
|
||||
> prefer `capture` for quick notes (auto-slug + dedupe), `put_page` for
|
||||
> full-control writes; if a narrowed token's list lacks capture, use `put_page`.
|
||||
> Why brains differ on the default:
|
||||
> [tutorial A1](../tutorials/connect-coding-agent.md#a1-on-the-host-serve-over-http).
|
||||
|
||||
## Remove
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ No server, no tunnel, no token needed. Works on both PGLite and Postgres engines
|
||||
`--surface verbs` exposes exactly the seven-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)) instead of the full catalog;
|
||||
`--surface starter` sits between (~26 ops: the verbs plus the daily-driver set);
|
||||
`--surface starter` sits between (~27 ops: the verbs plus the daily-driver set);
|
||||
omit the flag (default `full`) for every operation.
|
||||
|
||||
### Remote over OAuth 2.1 (recommended)
|
||||
|
||||
@@ -151,6 +151,9 @@ Stable phase names shipped in v0.15.2:
|
||||
writer adds chunks mid-run)
|
||||
- `repair_jsonb.run`, `repair_jsonb.<table>.<column>`
|
||||
- `backlinks.scan`
|
||||
- `backlinks.fix` — heartbeat-only (no total): the fix loop runs per-file
|
||||
locking + parse-validation + atomic writes, so agents see forward progress
|
||||
while it works through the gap list
|
||||
- `lint.pages`
|
||||
- `integrity.auto`
|
||||
- `eval.single`, `eval.ab`
|
||||
|
||||
@@ -82,8 +82,8 @@ each client.
|
||||
**Surface modes:** `--surface verbs` exposes EXACTLY the seven verbs —
|
||||
advertised list AND dispatch are filtered fail-closed (a hidden op returns
|
||||
`unknown_tool` even when called by name). `--surface starter` exposes the
|
||||
~26-op daily-driver set (`STARTER_OPS` in `src/mcp/surface.ts`): the seven
|
||||
verbs plus the daily brain-tool slice, the agent lane, `whoami`, and the
|
||||
~27-op daily-driver set (`STARTER_OPS` in `src/mcp/surface.ts`): the seven
|
||||
verbs plus the daily brain-tool slice, the agent lane, `whoami`, `capture`, and the
|
||||
`request_tools` discovery meta-op (re-derivable from production usage via
|
||||
`scripts/derive-starter-ops.ts`). Monotonic by construction: verbs ⊆ starter ⊆ full
|
||||
(pinned by test) — starter extends the ladder ABOVE verbs and never changes
|
||||
|
||||
@@ -204,7 +204,7 @@ gbrain schema add-alias researcher person
|
||||
|
||||
Read [`skills/conventions/schema-evolution.md`](../skills/conventions/schema-evolution.md) for the decision tree on when to add types vs aliases vs prefixes. The short version: <20 pages → don't pack-codify; 20-100 → alias on existing type; 100+ → first-class type.
|
||||
|
||||
**Lint your pack before shipping.** The 11-rule lint surface (with the optional `--with-db` flag for DB-aware checks) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
**Lint your pack before shipping.** The 14-rule lint surface (with the optional `--with-db` flag for DB-aware checks, including the stored-type alias/undeclared rules) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
|
||||
```bash
|
||||
gbrain schema lint --with-db
|
||||
|
||||
@@ -161,7 +161,7 @@ That's the whole wire-up. No token, no URL, no tunnel. The agent spawns
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md), frozen + additive-forever)
|
||||
instead of the full operation catalog, so the agent sees a tight, stable surface
|
||||
instead of a 110-tool wall. `--surface starter` sits between: the verbs plus the
|
||||
daily-driver set (~26 ops total). Drop the flag (or pass `--surface full`) for every
|
||||
daily-driver set (core page/search/graph ops + capture). Drop the flag (or pass `--surface full`) for every
|
||||
operation. The default when the flag is omitted is `full`, so existing wire-ups
|
||||
are unchanged.
|
||||
|
||||
@@ -244,7 +244,7 @@ habits to build. Your agent stops being amnesiac.
|
||||
| Agent "can't reach the brain" (Path A) | `gbrain serve --http` bound to loopback | Restart with `--bind 0.0.0.0` |
|
||||
| `list_skills` returns nothing / errors | Skill publishing OFF on the host | `gbrain config set mcp.publish_skills true` |
|
||||
| Token rejected on first call | Wrong/expired token | Re-mint with `gbrain auth create`; `--install` smoke-tests it for you |
|
||||
| `unknown tool: capture` | `capture` is CLI-only, not an MCP tool | Use `put_page` over MCP; `capture` only on the CLI |
|
||||
| `unknown tool: capture` | Your surface predates v0.47 or your token's surface was narrowed | Upgrade the host (capture is on starter + full now); on narrowed tokens use `put_page`, or `remember` on the verbs surface |
|
||||
| Empty results (Path B) | Brain has nothing in it yet | `gbrain import ~/notes/` or `gbrain capture "..."` |
|
||||
|
||||
## Next steps
|
||||
|
||||
+10
-5
@@ -213,7 +213,12 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
sites; `ctx.remote !== false` for untrust-unless-explicit-false). Don't default it falsy.
|
||||
- **Source isolation.** Every read-side op routes through `sourceScopeOpts(ctx)`; precedence
|
||||
is federated array (`ctx.auth.allowedSources`) > scalar (`ctx.sourceId`) > nothing. Don't
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak.
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak. Corollary
|
||||
(unscoped-check/scoped-write): `engine.getPage` with no opts matches ANY source while
|
||||
`putPage` defaults to `'default'` — an existence check + write pair must scope the read
|
||||
to the write's source (`getPage(slug, { sourceId: x ?? 'default' })`). Guarded by
|
||||
`scripts/check-getpage-scoped-write.mjs` (opt-out marker
|
||||
`gbrain-allow-unscoped-getpage` for read-only first-match sites).
|
||||
- **JSONB: never `JSON.stringify` into a `::jsonb` cast.** postgres.js double-encodes it (a jsonb
|
||||
string scalar); PGLite hides the bug. This bites BOTH spellings — the template form
|
||||
(`${JSON.stringify(x)}::jsonb`) AND the positional form (`executeRaw(\`…$N::jsonb\`, [JSON.stringify(x)])`,
|
||||
@@ -2936,7 +2941,7 @@ gbrain schema add-alias researcher person
|
||||
|
||||
Read [`skills/conventions/schema-evolution.md`](../skills/conventions/schema-evolution.md) for the decision tree on when to add types vs aliases vs prefixes. The short version: <20 pages → don't pack-codify; 20-100 → alias on existing type; 100+ → first-class type.
|
||||
|
||||
**Lint your pack before shipping.** The 11-rule lint surface (with the optional `--with-db` flag for DB-aware checks) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
**Lint your pack before shipping.** The 14-rule lint surface (with the optional `--with-db` flag for DB-aware checks, including the stored-type alias/undeclared rules) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
|
||||
```bash
|
||||
gbrain schema lint --with-db
|
||||
@@ -4139,7 +4144,7 @@ No server, no tunnel, no token needed. Works on both PGLite and Postgres engines
|
||||
`--surface verbs` exposes exactly the seven-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)) instead of the full catalog;
|
||||
`--surface starter` sits between (~26 ops: the verbs plus the daily-driver set);
|
||||
`--surface starter` sits between (~27 ops: the verbs plus the daily-driver set);
|
||||
omit the flag (default `full`) for every operation.
|
||||
|
||||
### Remote over OAuth 2.1 (recommended)
|
||||
@@ -4566,8 +4571,8 @@ each client.
|
||||
**Surface modes:** `--surface verbs` exposes EXACTLY the seven verbs —
|
||||
advertised list AND dispatch are filtered fail-closed (a hidden op returns
|
||||
`unknown_tool` even when called by name). `--surface starter` exposes the
|
||||
~26-op daily-driver set (`STARTER_OPS` in `src/mcp/surface.ts`): the seven
|
||||
verbs plus the daily brain-tool slice, the agent lane, `whoami`, and the
|
||||
~27-op daily-driver set (`STARTER_OPS` in `src/mcp/surface.ts`): the seven
|
||||
verbs plus the daily brain-tool slice, the agent lane, `whoami`, `capture`, and the
|
||||
`request_tools` discovery meta-op (re-derivable from production usage via
|
||||
`scripts/derive-starter-ops.ts`). Monotonic by construction: verbs ⊆ starter ⊆ full
|
||||
(pinned by test) — starter extends the ladder ABOVE verbs and never changes
|
||||
|
||||
@@ -31,7 +31,7 @@ Repo: https://github.com/garrytan/gbrain
|
||||
|
||||
## AI providers
|
||||
|
||||
- [docs/ai-providers/zeroentropy.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ai-providers/zeroentropy.md): ZeroEntropy zembed-1 embedding + zerank-2 reranker (hosted): API key, embedding switch, reranker config. (deprecated; hosted sunset 2026-09-04)
|
||||
- [docs/ai-providers/zeroentropy.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ai-providers/zeroentropy.md): ZeroEntropy (deprecated; hosted sunset 2026-09-04): the off-ramp for existing brains — migrate embeddings + reranker, self-host continuity, troubleshooting. Do not onboard.
|
||||
- [docs/ai-providers/llama-server-reranker.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ai-providers/llama-server-reranker.md): Local reranker via llama.cpp --reranking: Qwen3-Reranker or self-hosted ZE weights, --alias setup, gbrain config keys, cold-start timeout, budget-cap interaction.
|
||||
|
||||
## Debugging
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.12.2",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
|
||||
+2
-1
@@ -100,6 +100,7 @@
|
||||
"check:pagetype-exhaustive": "bash scripts/check-pagetype-exhaustive.sh",
|
||||
"check:pg-url-redaction": "bash scripts/check-pg-url-redaction.sh",
|
||||
"check:source-scope-onboard": "bash scripts/check-source-scope-onboard.sh",
|
||||
"check:getpage-scope": "node scripts/check-getpage-scoped-write.mjs",
|
||||
"postinstall": "bun run scripts/postinstall.ts",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin",
|
||||
@@ -167,7 +168,7 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.12.2",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.5",
|
||||
|
||||
+4
-3
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-plugin-tree-stamp: 0.46.10.0 -->
|
||||
<!-- gbrain-plugin-tree-stamp: 0.46.12.2 -->
|
||||
# gbrain plugin skill tree (generated — do not hand-edit)
|
||||
|
||||
This tree is the curated skill set for the gbrain Codex and Claude Code
|
||||
@@ -8,8 +8,9 @@ addition/exclusion).
|
||||
|
||||
## MCP surface note (read once)
|
||||
|
||||
The plugin's MCP server runs `gbrain serve --surface starter` — the 26-op
|
||||
daily-driver surface (the seven memory verbs + daily brain ops). 21
|
||||
The plugin's MCP server runs `gbrain serve --surface starter` — the
|
||||
27-op daily-driver surface (the seven memory verbs + daily
|
||||
brain ops + capture). 21
|
||||
bundled skills reference gbrain operations beyond that surface; every one of
|
||||
them has a first-class `gbrain` CLI path, which is the primary way skills
|
||||
drive gbrain. When a skill step names an operation your MCP tool list doesn't
|
||||
|
||||
@@ -174,8 +174,8 @@ get_job_progress ID
|
||||
```
|
||||
|
||||
Check structured result fields (exit code, stdout/stderr tails, attempts,
|
||||
timings) from `get_job`. Use `gbrain jobs stats` (CLI) for worker/queue
|
||||
health dashboard.
|
||||
timings) from `get_job`. Use `get_job_stats` (MCP) or `gbrain jobs stats`
|
||||
(CLI) for the worker/queue health dashboard incl. the wedged-queue signal.
|
||||
|
||||
### Control (MCP-callable)
|
||||
|
||||
@@ -236,6 +236,18 @@ Queue/priority/retry tuning is not exposed by `gbrain agent run`; submit the
|
||||
raw `subagent` handler via `gbrain jobs submit` (requires CLI trust) if you
|
||||
need those knobs.
|
||||
|
||||
**Admission control (v0.46.11.0).** Identical parentless `subagent` submits
|
||||
(same owner lane, payload, and execution options) coalesce onto the existing
|
||||
waiting job: `gbrain agent run` prints `coalesced` with the matched job id,
|
||||
and the `submit_agent` MCP response carries `coalesced: true`. Treat that as
|
||||
success — monitor the matched id, do NOT resubmit. Jobs still waiting after
|
||||
the TTL (48h default for `subagent`; `minions.ttl_waiting_hours.<name>`)
|
||||
are cancelled with reason prefix `waiting_ttl_expired`. If an operator has
|
||||
configured a waiting quota (`minions.quota_max_waiting.<name>`), a submit
|
||||
past the cap returns a structured, retryable `rate_limited` error — back
|
||||
off and check `gbrain jobs stats` for a `DIVERGENT QUEUE` line before
|
||||
retrying.
|
||||
|
||||
## Phase 2: Monitor
|
||||
|
||||
```
|
||||
@@ -488,6 +500,7 @@ Total tokens so far: 4.3k
|
||||
- Don't spawn a Minion for a single search query (use search tool directly)
|
||||
- Don't fire-and-forget without checking results
|
||||
- Don't spawn > 5 concurrent agents without checking `gbrain jobs stats` first
|
||||
- Don't resubmit when a submit reports `coalesced` — the work is already queued; monitor the matched job id instead
|
||||
- For subagent work, don't use `sessions_spawn` with `runtime: "subagent"` when Minions is available (use `gbrain agent run` instead)
|
||||
- Don't poll `get_job` in a tight loop (use `get_job_progress` for lightweight checks)
|
||||
- Don't run an operation expected to exceed ~2 minutes as a bare background shell — it dies with the session; route through the Durable execution ladder
|
||||
@@ -508,4 +521,5 @@ Total tokens so far: 4.3k
|
||||
- Replay a completed/failed job — `replay_job` (MCP)
|
||||
- Send sidechannel message — `send_job_message` (MCP)
|
||||
- Get structured progress — `get_job_progress` (MCP)
|
||||
- Queue stats — `gbrain jobs stats` (CLI; no MCP equivalent)
|
||||
- Queue stats — `get_job_stats` (MCP; admin scope over HTTP, same as the other
|
||||
jobs ops here — includes the wedged-queue silent-halt signal) or `gbrain jobs stats` (CLI)
|
||||
|
||||
@@ -177,8 +177,9 @@ Validate before sync:
|
||||
gbrain schema lint --with-db
|
||||
```
|
||||
|
||||
The `--with-db` flag opts into the 2 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`) that detect
|
||||
The `--with-db` flag opts into the 4 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`,
|
||||
`stored_type_is_alias`, `stored_type_undeclared`) that detect
|
||||
mis-declared types you'd otherwise discover only at runtime.
|
||||
|
||||
### Phase 5 — Sync (backfill existing pages with the new types)
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* CI guard for the unscoped-check/scoped-write source-isolation bug class.
|
||||
*
|
||||
* The trap: `engine.getPage(slug)` with NO opts matches the slug in ANY
|
||||
* source (first row wins), while the paired write (`putPage` /
|
||||
* `importFromContent` / `tx.putPage`) defaults to the 'default' source. A
|
||||
* page that exists only in source B makes the existence check "succeed",
|
||||
* and the write then targets a DIFFERENT row — duplicates, clobbers, or
|
||||
* crashes (this class broke dream cycles for weeks; the writer/slug-registry
|
||||
* variant forced spurious slug disambiguation).
|
||||
*
|
||||
* Heuristic (deliberately file-scoped, same posture as
|
||||
* check-source-scope-onboard.sh): flag any non-test source file that contains
|
||||
* BOTH
|
||||
* (a) a getPage/tx.getPage call whose balanced argument span has no second
|
||||
* argument at all, OR a conditional second argument whose false branch
|
||||
* is undefined/null/{} — shorthand (`x ? { sourceId } : undefined`) and
|
||||
* expanded (`x ? { sourceId: x } : undefined`) forms alike (any-source
|
||||
* when unset — the read half of the bug),
|
||||
* AND
|
||||
* (b) any write-path call: putPage( / importFromContent( / importFromFile(.
|
||||
*
|
||||
* The fix pattern (operations.ts): `getPage(slug, { sourceId: x ?? 'default' })`
|
||||
* — mirror the write's schema default on the read.
|
||||
*
|
||||
* Opt-out: a `gbrain-allow-unscoped-getpage: <reason>` comment ANYWHERE in the
|
||||
* getPage call span or on the line above it (for genuinely read-only,
|
||||
* first-match-semantics callers).
|
||||
*
|
||||
* Exit 0 = clean, 1 = violations. Runs under node or bun.
|
||||
*/
|
||||
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
// Default scan roots; overridable via argv so the guard's self-test can point
|
||||
// it at fixtures (`node check-getpage-scoped-write.mjs /tmp/fixtures`).
|
||||
const ROOTS = process.argv.slice(2).length > 0 ? process.argv.slice(2) : ['src'];
|
||||
|
||||
const GETPAGE_RE = /\.\s*getPage\s*(?:<[^>;]*>)?\s*\(/g;
|
||||
const WRITE_RE = /\b(putPage|importFromContent|importFromFile)\s*(?:<[^>;]*>)?\s*\(/;
|
||||
const OPT_OUT = 'gbrain-allow-unscoped-getpage';
|
||||
|
||||
/** Walk from the '(' at openIdx and return [start,end) of the balanced span,
|
||||
* respecting strings, template literals, and comments. */
|
||||
function findSpan(src, openIdx) {
|
||||
let depth = 0;
|
||||
let mode = 'code'; // code | line | block | sq | dq | tpl
|
||||
for (let i = openIdx; i < src.length; i++) {
|
||||
const c = src[i];
|
||||
const n = src[i + 1];
|
||||
if (mode === 'line') { if (c === '\n') mode = 'code'; continue; }
|
||||
if (mode === 'block') { if (c === '*' && n === '/') { mode = 'code'; i++; } continue; }
|
||||
if (mode === 'sq') { if (c === '\\') { i++; continue; } if (c === "'") mode = 'code'; continue; }
|
||||
if (mode === 'dq') { if (c === '\\') { i++; continue; } if (c === '"') mode = 'code'; continue; }
|
||||
if (mode === 'tpl') { if (c === '\\') { i++; continue; } if (c === '`') mode = 'code'; continue; }
|
||||
if (c === '/' && n === '/') { mode = 'line'; i++; continue; }
|
||||
if (c === '/' && n === '*') { mode = 'block'; i++; continue; }
|
||||
if (c === "'") { mode = 'sq'; continue; }
|
||||
if (c === '"') { mode = 'dq'; continue; }
|
||||
if (c === '`') { mode = 'tpl'; continue; }
|
||||
if (c === '(') depth++;
|
||||
else if (c === ')') { depth--; if (depth === 0) return [openIdx + 1, i]; }
|
||||
}
|
||||
return [openIdx + 1, src.length];
|
||||
}
|
||||
|
||||
/** Blank out comments so commented examples don't trip the probes. */
|
||||
function stripComments(s) {
|
||||
return s.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
}
|
||||
|
||||
/** Split a balanced span into top-level arguments (commas at depth 0 only). */
|
||||
function topLevelArgs(span) {
|
||||
const args = [];
|
||||
let depth = 0;
|
||||
let mode = 'code';
|
||||
let cur = '';
|
||||
for (let i = 0; i < span.length; i++) {
|
||||
const c = span[i];
|
||||
const n = span[i + 1];
|
||||
if (mode === 'line') { if (c === '\n') mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'block') { if (c === '*' && n === '/') { mode = 'code'; cur += '*/'; i++; continue; } cur += c; continue; }
|
||||
if (mode === 'sq') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === "'") mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'dq') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === '"') mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'tpl') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === '`') mode = 'code'; cur += c; continue; }
|
||||
if (c === '/' && n === '/') { mode = 'line'; cur += c; continue; }
|
||||
if (c === '/' && n === '*') { mode = 'block'; cur += c; continue; }
|
||||
if (c === "'") { mode = 'sq'; cur += c; continue; }
|
||||
if (c === '"') { mode = 'dq'; cur += c; continue; }
|
||||
if (c === '`') { mode = 'tpl'; cur += c; continue; }
|
||||
if (c === '(' || c === '[' || c === '{') depth++;
|
||||
else if (c === ')' || c === ']' || c === '}') depth--;
|
||||
else if (c === ',' && depth === 0) { args.push(cur); cur = ''; continue; }
|
||||
cur += c;
|
||||
}
|
||||
if (cur.trim().length > 0) args.push(cur);
|
||||
return args;
|
||||
}
|
||||
|
||||
/** True when the getPage second argument is the any-source-when-unset shape. */
|
||||
function isUnscopedRead(span) {
|
||||
const args = topLevelArgs(span);
|
||||
if (args.length < 2) return true; // no opts at all → unscoped
|
||||
const opts = stripComments(args[1]).trim();
|
||||
// Ternary opts whose false branch is undefined/null/{} — any-source when
|
||||
// unset. Covers BOTH the shorthand (`x ? { sourceId } : undefined`) and the
|
||||
// expanded form (`x ? { sourceId: x } : undefined`): the object-literal
|
||||
// colon in the expanded form defeated a naive [^:]* regex, so this checks
|
||||
// "mentions sourceId + ends in a bare-empty false branch" instead.
|
||||
if (opts.includes('sourceId') && /\?[\s\S]*:\s*(undefined|null|\{\s*\})\s*$/.test(opts)) return true;
|
||||
if (/^(undefined|null|\{\s*\})$/.test(opts)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
|
||||
function scanFile(file) {
|
||||
const src = readFileSync(file, 'utf8');
|
||||
if (!WRITE_RE.test(stripComments(src))) return; // no write path in this file → read-only semantics allowed
|
||||
GETPAGE_RE.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = GETPAGE_RE.exec(src))) {
|
||||
const openIdx = m.index + m[0].length - 1;
|
||||
const [s, e] = findSpan(src, openIdx);
|
||||
const span = src.slice(s, e);
|
||||
// Opt-out marker inside the span, on the lines just before the call, or
|
||||
// in a trailing comment on the closing-paren line.
|
||||
const before = src.slice(Math.max(0, m.index - 300), m.index);
|
||||
const afterEnd = src.indexOf('\n', e);
|
||||
const tail = src.slice(e, afterEnd === -1 ? src.length : afterEnd);
|
||||
if (
|
||||
span.includes(OPT_OUT) ||
|
||||
before.split('\n').slice(-3).join('\n').includes(OPT_OUT) ||
|
||||
tail.includes(OPT_OUT)
|
||||
) continue;
|
||||
if (!isUnscopedRead(span)) continue;
|
||||
const line = src.slice(0, m.index).split('\n').length;
|
||||
violations.push(
|
||||
`${file}:${line} unscoped getPage(...) in a file that also writes (putPage/importFromContent) — ` +
|
||||
`scope the read to the write's source: getPage(slug, { sourceId: x ?? 'default' })`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function walk(dir) {
|
||||
let ents;
|
||||
try { ents = readdirSync(dir); } catch { return; }
|
||||
for (const ent of ents) {
|
||||
if (ent === 'node_modules') continue;
|
||||
const p = join(dir, ent);
|
||||
const st = statSync(p);
|
||||
if (st.isDirectory()) walk(p);
|
||||
else if (p.endsWith('.ts') && !p.endsWith('.test.ts')) scanFile(p);
|
||||
}
|
||||
}
|
||||
|
||||
for (const root of ROOTS) walk(root);
|
||||
|
||||
if (violations.length) {
|
||||
console.error('Unscoped-getPage-with-write violations (source-isolation bug class):\n');
|
||||
for (const v of violations) console.error(' ' + v);
|
||||
console.error(
|
||||
`\n${violations.length} violation(s). Fix: pass { sourceId: x ?? 'default' } on the read ` +
|
||||
`(mirrors putPage's schema default), or mark genuinely read-only first-match calls with ` +
|
||||
`a '${OPT_OUT}: <reason>' comment.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('check-getpage-scoped-write: clean (no unscoped getPage in write-path files)');
|
||||
@@ -36,6 +36,19 @@ const EXTRA_FLAGS: Record<string, string[]> = {
|
||||
sync: ['--pace', '--pace-max-concurrency'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Modules the import scan must SKIP. thin-client-routing.ts is a pure router —
|
||||
* its flag literals belong to the commands it routes (takes/search/jobs/cache/
|
||||
* quarantine), and each of those declares its own flags in its own case block;
|
||||
* scanning the router bleeds takes/quarantine flags into jobs (whose case
|
||||
* block imports it for the `jobs stats` thin-client route).
|
||||
*/
|
||||
const EXCLUDED_MODULES = ['thin-client-routing.ts'];
|
||||
|
||||
function isExcludedModule(p: string): boolean {
|
||||
return EXCLUDED_MODULES.some(m => p.endsWith(`/${m}`));
|
||||
}
|
||||
|
||||
/** Universal helper flags every command may see (parsed or short-circuited upstream). */
|
||||
const UNIVERSAL_FLAGS = ['--help', '--json', '--brain', '--source'];
|
||||
|
||||
@@ -59,7 +72,7 @@ function relativeImports(src: string, fromDir: string): string[] {
|
||||
for (const m of src.matchAll(/import\('(\.\.?\/[^']+\.ts)'\)/g)) paths.add(m[1]);
|
||||
return [...paths]
|
||||
.map(p => resolvePath(fromDir, p))
|
||||
.filter(p => existsSync(p))
|
||||
.filter(p => existsSync(p) && !isExcludedModule(p))
|
||||
.flatMap(p => [p, ...facadeExpansion(p)]);
|
||||
}
|
||||
|
||||
@@ -166,7 +179,7 @@ export function buildFlagRegistry(): Record<string, string[]> {
|
||||
// own ./relative imports.
|
||||
const commandModules = [...block.matchAll(/import\('(\.\/[^']+\.ts)'\)/g)]
|
||||
.map(mm => resolvePath(join(ROOT, 'src'), mm[1]))
|
||||
.filter(p => existsSync(p));
|
||||
.filter(p => existsSync(p) && !isExcludedModule(p));
|
||||
for (const modPath of commandModules) {
|
||||
// A command module that IS a peeled façade counts its module files as
|
||||
// part of itself: their text scans at module depth and THEIR relative
|
||||
|
||||
@@ -179,8 +179,9 @@ addition/exclusion).
|
||||
|
||||
## MCP surface note (read once)
|
||||
|
||||
The plugin's MCP server runs \`gbrain serve --surface starter\` — the 26-op
|
||||
daily-driver surface (the seven memory verbs + daily brain ops). ${gapSkills}
|
||||
The plugin's MCP server runs \`gbrain serve --surface starter\` — the
|
||||
${STARTER_OPS.size}-op daily-driver surface (the seven memory verbs + daily
|
||||
brain ops + capture). ${gapSkills}
|
||||
bundled skills reference gbrain operations beyond that surface; every one of
|
||||
them has a first-class \`gbrain\` CLI path, which is the primary way skills
|
||||
drive gbrain. When a skill step names an operation your MCP tool list doesn't
|
||||
|
||||
@@ -66,3 +66,4 @@ check-module-size.sh scanner yes committed per-file line ceilings (module-size-l
|
||||
check-structural-manifest.sh buildfresh exempt regenerate+diff of structural-suites.tsv (classify-tests.ts); the diff IS the self-test
|
||||
check-opencode-pin.sh repostate exempt pin-stamp drift check (OPENCODE-CLI-PIN.md stamps vs heavy-tests opencode-door env); own bun guard tests in test/check-bootstrap-guards.test.ts
|
||||
check-pin-doc-privacy.sh repostate exempt PIN-doc placeholder discipline (no operator paths/key material/emails in docs/mcp/*-CLI-PIN.md); own bun guard tests in test/check-bootstrap-guards.test.ts
|
||||
check-getpage-scoped-write.mjs scanner yes unscoped-getPage + write co-occurrence scanner (source-isolation bug class); argv root override; fixtures under test/fixtures/guards/; also in verify CHECKS
|
||||
|
||||
|
@@ -176,7 +176,7 @@ export const SECTIONS: DocSection[] = [
|
||||
{
|
||||
title: "docs/ai-providers/zeroentropy.md",
|
||||
description:
|
||||
"ZeroEntropy zembed-1 embedding + zerank-2 reranker (hosted): API key, embedding switch, reranker config. (deprecated; hosted sunset 2026-09-04)",
|
||||
"ZeroEntropy (deprecated; hosted sunset 2026-09-04): the off-ramp for existing brains — migrate embeddings + reranker, self-host continuity, troubleshooting. Do not onboard.",
|
||||
path: "docs/ai-providers/zeroentropy.md",
|
||||
// Setup walkthrough — discoverable in the index, not inlined in the
|
||||
// single-fetch bundle (keeps llms-full.txt under FULL_SIZE_BUDGET).
|
||||
|
||||
@@ -3,33 +3,33 @@
|
||||
# Raising a ceiling is a conscious, reviewer-visible act. Lower ceilings in
|
||||
# the same commit as any peel (the guard fails on >50 lines of stale slack).
|
||||
# Columns: path max_lines policy note
|
||||
src/commands/doctor.ts 4205 ratchet peel target: containment sprint C8-C13
|
||||
src/commands/doctor.ts 4270 ratchet peel target: containment sprint C8-C13; grown v0.46.11.0 five-issue wave
|
||||
src/core/operations.ts 303 ratchet peel target: containment sprint C4-C7
|
||||
src/core/postgres-engine.ts 5734 ratchet peel target: containment sprint C15
|
||||
src/core/pglite-engine.ts 5625 ratchet peel target: containment sprint C15
|
||||
src/core/postgres-engine.ts 5770 ratchet peel target: containment sprint C15; grown v0.46.11.0 five-issue wave
|
||||
src/core/pglite-engine.ts 5660 ratchet peel target: containment sprint C15; grown v0.46.11.0 five-issue wave
|
||||
src/core/migrate.ts 668 region-exempt append-only MIGRATIONS array grows freely; runner logic is ratcheted
|
||||
src/commands/sync.ts 4121 ratchet peel target: containment sprint C13-C14
|
||||
src/core/ai/gateway.ts 4116 ratchet watchlist
|
||||
src/cli.ts 3337 ratchet watchlist
|
||||
src/commands/sync.ts 4300 ratchet peel target: containment sprint C13-C14; grown v0.46.11.0 five-issue wave
|
||||
src/core/ai/gateway.ts 4117 ratchet watchlist
|
||||
src/cli.ts 3385 ratchet watchlist; +21 gap-closure wave: thin-client routing call sites (logic in commands/thin-client-routing.ts)
|
||||
src/core/cycle.ts 2933 ratchet
|
||||
src/commands/serve-http.ts 2836 ratchet
|
||||
src/commands/jobs.ts 2839 ratchet
|
||||
src/commands/jobs.ts 2950 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/core/search/hybrid.ts 2479 ratchet
|
||||
src/core/engine.ts 2343 ratchet
|
||||
src/commands/autopilot.ts 2301 ratchet
|
||||
src/commands/extract.ts 2161 ratchet
|
||||
src/commands/extract-conversation-facts.ts 1968 ratchet
|
||||
src/core/import-file.ts 1904 ratchet
|
||||
src/core/cycle/synthesize.ts 2616 ratchet
|
||||
src/core/import-file.ts 2000 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/core/cycle/synthesize.ts 2685 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/commands/embed.ts 1963 ratchet
|
||||
src/core/types.ts 1829 ratchet
|
||||
src/commands/skillpack.ts 1763 ratchet
|
||||
src/core/minions/queue.ts 1824 ratchet
|
||||
src/core/minions/queue.ts 2130 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/commands/init.ts 1932 ratchet
|
||||
src/commands/integrations.ts 1675 ratchet
|
||||
src/core/minions/handlers/subagent.ts 1643 ratchet
|
||||
src/commands/bootstrap.ts 1923 ratchet grandfathered at merge (grew past the 1500 cap on master)
|
||||
src/core/minions/worker.ts 1508 ratchet grandfathered at merge (grew past the 1500 cap on master, #4170)
|
||||
src/core/minions/worker.ts 1560 ratchet grandfathered at merge (grew past the 1500 cap on master, #4170); grown v0.46.11.0 five-issue wave
|
||||
src/commands/sources.ts 1586 ratchet
|
||||
src/core/bootstrap/harness.ts 1947 ratchet
|
||||
src/commands/hook.ts 1525 ratchet
|
||||
|
||||
|
@@ -94,6 +94,7 @@ CHECKS=(
|
||||
"check:doc-history"
|
||||
"check:fixture-privacy"
|
||||
"check:source-scope-onboard"
|
||||
"check:getpage-scope"
|
||||
"check:no-double-retry"
|
||||
"check:batch-audit-site"
|
||||
"check:engine-dynamic-import"
|
||||
|
||||
@@ -103,6 +103,7 @@ test/features.test.ts CLI routing 2 bun-file
|
||||
test/filing-rules-resolution.serial.test.ts per-source filing-rules resolution 3 readFileSync
|
||||
test/fix-wave-structural.test.ts #2084 — cli.ts owns process-exit teardown via finishCliTeardown 4 readFileSync
|
||||
test/fix-wave-structural.test.ts WAL-repair wave structural pins (#223/#2575) 4 readFileSync
|
||||
test/fix-wave-structural.test.ts five-issue fix wave — integrity progress is (source_id, slug)-keyed 1 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1077 — admin register-client supports PKCE public clients 1 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1090 — admin embed two-tier resolution 3 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1124 — query --no-expand actually negates expand 1 readFileSync
|
||||
|
||||
|
Can't render this file because it contains an unexpected character in line 27 and column 63.
|
@@ -1,17 +1,19 @@
|
||||
---
|
||||
feature_pitch: ZeroEntropy zembed-1 embeddings + zerank-2 cross-encoder reranking
|
||||
feature_pitch: "HISTORICAL: ZeroEntropy zembed-1 embeddings + zerank-2 reranking (provider retired 2026-09-04)"
|
||||
required_action: no # purely opt-in
|
||||
---
|
||||
|
||||
# v0.35.0.0 migration notes
|
||||
|
||||
> **HISTORICAL — DO NOT FOLLOW.** ZeroEntropy's hosted API shuts down
|
||||
> **2026-09-04**; the opt-in instructions below would strand a brain on a dead
|
||||
> provider. To LEAVE ZeroEntropy, follow `skills/migrations/v0.46.3.0.md`.
|
||||
> **HISTORICAL — DO NOT FOLLOW. Do not execute any command in this file.**
|
||||
> ZeroEntropy's hosted API shuts down **2026-09-04**; the opt-in config
|
||||
> edits and commands below would strand a brain on a dead provider. To
|
||||
> LEAVE ZeroEntropy, follow `skills/migrations/v0.46.3.0.md`.
|
||||
|
||||
ZeroEntropy support landed. **No required user action.** Reranker is on by
|
||||
default for `tokenmax` mode only; embedding model is unchanged for everyone
|
||||
unless the user explicitly opts in via config file or env var.
|
||||
ZeroEntropy support landed (historical record). **No required user action.**
|
||||
Reranker was on by default for `tokenmax` mode only; the embedding model was
|
||||
unchanged for everyone unless the user explicitly opted in via config file
|
||||
or env var.
|
||||
|
||||
## What changed automatically
|
||||
|
||||
@@ -28,43 +30,23 @@ unless the user explicitly opts in via config file or env var.
|
||||
- `conservative` and `balanced` modes default reranker = false. Nothing
|
||||
changes for those users without an explicit opt-in.
|
||||
|
||||
## What the user can do (optional)
|
||||
## What the user could do at the time (historical — do not run any of this)
|
||||
|
||||
### Try zembed-1 embeddings
|
||||
### The zembed-1 opt-in (era recipe, now a strand-your-brain trap)
|
||||
|
||||
Switching embedding models invalidates the vector index — you'll need to
|
||||
re-embed. Edit `~/.gbrain/config.json`:
|
||||
The era's opt-in was a config-file edit pointing `embedding_model` at
|
||||
`zeroentropyai:zembed-1` (valid Matryoshka dims: 2560, 1280, 640, 320,
|
||||
160, 80, 40), followed by a key export and a staged re-embed
|
||||
(`gbrain models doctor`, a small `--stale` smoke, then the full pass).
|
||||
Running that today points a brain at an API that dies 2026-09-04 — the
|
||||
maintained path is the off-ramp in `skills/migrations/v0.46.3.0.md`.
|
||||
|
||||
```json
|
||||
{
|
||||
"embedding_model": "zeroentropyai:zembed-1",
|
||||
"embedding_dimensions": 2560
|
||||
}
|
||||
```
|
||||
### The zerank-2 opt-in on conservative/balanced (era recipe)
|
||||
|
||||
Valid dims: 2560, 1280, 640, 320, 160, 80, 40 (Matryoshka-style; smaller
|
||||
trades quality for storage). Then:
|
||||
|
||||
```bash
|
||||
export ZEROENTROPY_API_KEY=...
|
||||
gbrain models doctor # verify config
|
||||
gbrain embed --stale --limit 50 # smoke a small re-embed
|
||||
gbrain embed --stale # full re-embed
|
||||
```
|
||||
|
||||
### Try zerank-2 on conservative/balanced
|
||||
|
||||
```bash
|
||||
export ZEROENTROPY_API_KEY=...
|
||||
gbrain config set search.reranker.enabled true
|
||||
gbrain models doctor # verify reranker_config + reachability
|
||||
gbrain query "some query that previously misranked"
|
||||
```
|
||||
|
||||
To opt out:
|
||||
```bash
|
||||
gbrain config set search.reranker.enabled false
|
||||
```
|
||||
The era's opt-in was `search.reranker.enabled true` with a ZE key set,
|
||||
verified via `gbrain models doctor`. Enabling reranking TODAY should set
|
||||
the surviving model first: `gbrain config set search.reranker.model
|
||||
voyage:rerank-2.5`, then enable.
|
||||
|
||||
### Opt out of reranker on tokenmax
|
||||
|
||||
|
||||
@@ -1,45 +1,40 @@
|
||||
---
|
||||
name: v0.36.2.0
|
||||
version: 0.36.2.0
|
||||
headline: ZeroEntropy is the new default for embedding + reranker
|
||||
headline: "HISTORICAL: ZeroEntropy was briefly the default for embedding + reranker"
|
||||
---
|
||||
|
||||
# v0.36.2.0 Migration: ZeroEntropy as default
|
||||
|
||||
> **HISTORICAL — DO NOT FOLLOW.** ZeroEntropy's hosted API shuts down
|
||||
> **2026-09-04** and `gbrain ze-switch` now refuses forward use. To LEAVE
|
||||
> ZeroEntropy, follow `skills/migrations/v0.46.3.0.md`.
|
||||
> **HISTORICAL — DO NOT FOLLOW. Do not execute any command in this file.**
|
||||
> ZeroEntropy's hosted API shuts down **2026-09-04**; every ze-switch
|
||||
> command below exits 1 with `reason: provider_sunset` today (the command
|
||||
> is a pure refusal/redirect shim), and the benchmark pitch below is
|
||||
> obsolete. To LEAVE ZeroEntropy, follow `skills/migrations/v0.46.3.0.md`.
|
||||
|
||||
**TL;DR:** v0.36.2.0 makes ZeroEntropy (`zembed-1` at 1280d via Matryoshka, `zerank-2` cross-encoder reranker) the default retrieval stack. Real-corpus benchmark says: 2.2× faster than OpenAI, 2.6× cheaper at regular pricing, wins 11/20 head-to-head queries, reranker reshuffles 60% of top-1 results. New installs land on it automatically. Existing brains see a one-shot upgrade prompt with a cost estimate; default-on-Enter is "stay on current provider" so the safe path is the no-action path.
|
||||
**TL;DR (historical record):** v0.36.2.0 made ZeroEntropy (`zembed-1` at 1280d via Matryoshka, `zerank-2` cross-encoder reranker) the default retrieval stack, on the strength of a real-corpus benchmark that has since been overtaken by events — the provider announced its shutdown. New installs landed on it automatically. Existing brains saw a one-shot upgrade prompt with a cost estimate; default-on-Enter was "stay on current provider".
|
||||
|
||||
This migration is **opt-in for existing brains**. Running `gbrain upgrade` does not silently re-embed your data. The prompt is TTY-only; non-TTY upgrades (CI, cron, docker) print an informational stderr line and skip the switch.
|
||||
This migration was **opt-in for existing brains**. Running `gbrain upgrade` did not silently re-embed data. The prompt was TTY-only; non-TTY upgrades (CI, cron, docker) printed an informational stderr line and skipped the switch.
|
||||
|
||||
## What the agent should do post-upgrade
|
||||
## What the agent did post-upgrade (v0.36.2 era — every command here exits 1 today)
|
||||
|
||||
When the user invokes any gbrain command immediately after `gbrain upgrade` to v0.36.2.0:
|
||||
When a user invoked any gbrain command immediately after `gbrain upgrade` to v0.36.2.0, the playbook was:
|
||||
|
||||
1. **Check whether the prompt has been surfaced yet.** `gbrain ze-switch --dry-run --json` shows the planner state including `ze_switch_offered`, `ze_switch_prompt_shown`, and the comparison numbers. If `ze_switch_offered` is true and `ze_switch_prompt_shown` is false, the user hasn't seen the prompt yet.
|
||||
1. The agent checked planner state via `ze-switch --dry-run --json` (`ze_switch_offered`, `ze_switch_prompt_shown`, the comparison numbers).
|
||||
|
||||
2. **Surface the comparison numbers.** Quote the benchmark from the prompt:
|
||||
- Wins 11/20 queries head-to-head (OpenAI 6, Voyage 4)
|
||||
- 442ms avg vs OpenAI 973ms (2.2× faster)
|
||||
- $0.05/M tokens vs OpenAI $0.13 (2.6× cheaper at regular pricing)
|
||||
- zerank-2 reshuffles 60% of top-1 results
|
||||
- Only 10-18% overlap between providers — pairing primary + reranker compounds
|
||||
2. It surfaced the era's benchmark numbers (head-to-head query wins, latency, price, reranker reshuffle rate, provider overlap).
|
||||
|
||||
3. **Recommend switching** unless the user has stated a constraint that makes ZE unsuitable (no internet, regulatory concern with a specific provider, contractual commitment to another stack). When recommending, point them at:
|
||||
```bash
|
||||
gbrain ze-switch # interactive prompt
|
||||
gbrain ze-switch --dry-run # see the plan without applying
|
||||
```
|
||||
3. It recommended switching unless the user had a constraint making ZE unsuitable, pointing at the interactive prompt and the dry-run plan.
|
||||
|
||||
4. **If the user already declined** (`ze_switch_already_declined` true) and they ask about it later, mention `gbrain ze-switch --force` re-opens the prompt. The 90-day decline window auto-resets after that, so a year-later contributor "we have better benchmarks now" data gets surfaced naturally.
|
||||
4. For a prior decline, `--force` re-opened the prompt (90-day re-ask window).
|
||||
|
||||
5. **If the user switched and now regrets it**, mention `gbrain ze-switch --undo`. Restores their prior model + dim + reranker state with a symmetric cost-warning prompt (re-embedding back is also a real cost).
|
||||
5. For regret after switching, `--undo` restored the prior model + dim + reranker state behind a cost-warning prompt.
|
||||
|
||||
## The CLI surface in full
|
||||
None of that flow exists anymore: the recommendation aged into a liability when the shutdown was announced, and today the entire surface refuses.
|
||||
|
||||
| Command | Effect |
|
||||
## The CLI surface in full (historical — every row exits 1 with `provider_sunset` today)
|
||||
|
||||
| Command | Effect (v0.36.2 era) |
|
||||
|---|---|
|
||||
| `gbrain ze-switch` | Interactive prompt (TTY only). Default-on-Enter = stay. |
|
||||
| `gbrain ze-switch --dry-run` | Print plan as text. Change nothing. |
|
||||
@@ -48,8 +43,8 @@ When the user invokes any gbrain command immediately after `gbrain upgrade` to v
|
||||
| `gbrain ze-switch --non-interactive --ignore-missing-key` | Same, but stage the schema change before the key is ready. Embeddings fail loud until key arrives. |
|
||||
| `gbrain ze-switch --resume` | Complete a half-applied switch (crash recovery). |
|
||||
| `gbrain ze-switch --force` | Bypass the prompt-shown gate (re-show after `n`). |
|
||||
| `gbrain ze-switch --undo` | Reverse with cost-warning prompt. |
|
||||
| `gbrain ze-switch --undo --non-interactive --confirm-reembed` | Scripted undo. The `--confirm-reembed` flag is required (un-doing also pays for re-embed). |
|
||||
| `gbrain ze-switch --undo` | Reverse with cost-warning prompt. (Today: prints the return-path `gbrain migrate embeddings` command instead of acting.) |
|
||||
| `gbrain ze-switch --undo --non-interactive --confirm-reembed` | Scripted undo. (Today: prints guidance, exit 1.) |
|
||||
|
||||
## Consolidation with the v0.32.7 chunker prompt
|
||||
|
||||
@@ -59,8 +54,8 @@ If a brain has BOTH a stale chunker version AND the ZE-switch offered, the `Retr
|
||||
|
||||
`gbrain doctor` now runs two new ZE-aware checks:
|
||||
|
||||
- **`ze_embedding_health`** — warns if `embedding_model` starts with `zeroentropyai:` but no key is configured (neither env nor `gbrain config set zeroentropy_api_key`). Fix hint points at the setup URL.
|
||||
- **`embedding_width_consistency`** — asserts the configured `embedding_dimensions` matches the actual `vector(N)` width on `content_chunks.embedding`. Warns on drift. Fix hint suggests `gbrain ze-switch --resume` if drift came from a half-applied switch, or `gbrain config set embedding_dimensions <schema-dim>` to match the existing schema.
|
||||
- **`ze_embedding_health`** — warns if `embedding_model` starts with `zeroentropyai:` but no key is configured. (At the time the fix hint pointed at the setup URL; today it points at the migration off-ramp.)
|
||||
- **`embedding_width_consistency`** — asserts the configured `embedding_dimensions` matches the actual `vector(N)` width on `content_chunks.embedding`. Warns on drift. (At the time the fix hint suggested `--resume`; today the check prints an engine-branched recovery recipe — there is no resume.)
|
||||
|
||||
## What changed under the hood
|
||||
|
||||
@@ -70,21 +65,21 @@ If a brain has BOTH a stale chunker version AND the ZE-switch offered, the `Retr
|
||||
- Schema transition (when user accepts the switch): DROP indexes → ALTER `content_chunks.embedding` to `vector(1280)` → CREATE INDEX. Atomic inside one `engine.transaction()`. HNSW indexes recreated in the same transaction; no silent slow-search window.
|
||||
- Three new config keys: `ze_switch_prompt_shown`, `ze_switch_requested`, `ze_switch_applied`. Plus `ze_switch_previous_snapshot` (JSON, captures prior config for `--undo`) and `ze_switch_declined_at` (ISO timestamp for the 90-day re-ask gate).
|
||||
|
||||
## What NOT to do
|
||||
## What NOT to do (historical guardrails for the era's flow)
|
||||
|
||||
- Don't run `gbrain ze-switch --non-interactive --ignore-missing-key` for a user without explaining the consequence — every embed call will fail until they set `ZEROENTROPY_API_KEY`. Surface that loudly.
|
||||
- Don't tell the user the sale price ($0.025/M) is the cost they'll pay long-term. It's a promotional rate. The CHANGELOG and prompt cite the regular $0.05/M as the cost anchor.
|
||||
- Don't assume the user wants the switch because the comparison numbers favor it. The user owns the decision. The prompt's default-on-Enter is "stay" for exactly this reason.
|
||||
- The `--ignore-missing-key` staging spelling required explaining that every embed call would fail until the key arrived.
|
||||
- The sale price was promotional; the regular price was the cost anchor.
|
||||
- The user owned the switch decision; the prompt's default-on-Enter was "stay" for exactly this reason.
|
||||
|
||||
## Why 1280d, not 1024d
|
||||
|
||||
The valid ZE Matryoshka dim steps are `2560, 1280, 640, 320, 160, 80, 40`. 1024 (Voyage's step) is NOT on ZE's list — see `src/core/ai/dims.ts:ZEROENTROPY_VALID_DIMS`. 1280 is the step closest to the prior OpenAI 1536d default and stays in the high-recall zone of the Matryoshka curve.
|
||||
|
||||
## Verifying the switch
|
||||
## Verifying the switch (historical — the ze-switch line exits 1 today)
|
||||
|
||||
```bash
|
||||
```text
|
||||
gbrain doctor # both new checks should be green
|
||||
gbrain ze-switch --dry-run # status should be skipped_already_applied
|
||||
gbrain ze-switch --dry-run # (era) status skipped_already_applied; (today) refuses
|
||||
gbrain models # confirm embedding + reranker defaults
|
||||
gbrain search "test query" --limit 5 # confirm the reranker is firing
|
||||
```
|
||||
|
||||
@@ -174,8 +174,8 @@ get_job_progress ID
|
||||
```
|
||||
|
||||
Check structured result fields (exit code, stdout/stderr tails, attempts,
|
||||
timings) from `get_job`. Use `gbrain jobs stats` (CLI) for worker/queue
|
||||
health dashboard.
|
||||
timings) from `get_job`. Use `get_job_stats` (MCP) or `gbrain jobs stats`
|
||||
(CLI) for the worker/queue health dashboard incl. the wedged-queue signal.
|
||||
|
||||
### Control (MCP-callable)
|
||||
|
||||
@@ -236,6 +236,18 @@ Queue/priority/retry tuning is not exposed by `gbrain agent run`; submit the
|
||||
raw `subagent` handler via `gbrain jobs submit` (requires CLI trust) if you
|
||||
need those knobs.
|
||||
|
||||
**Admission control (v0.46.11.0).** Identical parentless `subagent` submits
|
||||
(same owner lane, payload, and execution options) coalesce onto the existing
|
||||
waiting job: `gbrain agent run` prints `coalesced` with the matched job id,
|
||||
and the `submit_agent` MCP response carries `coalesced: true`. Treat that as
|
||||
success — monitor the matched id, do NOT resubmit. Jobs still waiting after
|
||||
the TTL (48h default for `subagent`; `minions.ttl_waiting_hours.<name>`)
|
||||
are cancelled with reason prefix `waiting_ttl_expired`. If an operator has
|
||||
configured a waiting quota (`minions.quota_max_waiting.<name>`), a submit
|
||||
past the cap returns a structured, retryable `rate_limited` error — back
|
||||
off and check `gbrain jobs stats` for a `DIVERGENT QUEUE` line before
|
||||
retrying.
|
||||
|
||||
## Phase 2: Monitor
|
||||
|
||||
```
|
||||
@@ -488,6 +500,7 @@ Total tokens so far: 4.3k
|
||||
- Don't spawn a Minion for a single search query (use search tool directly)
|
||||
- Don't fire-and-forget without checking results
|
||||
- Don't spawn > 5 concurrent agents without checking `gbrain jobs stats` first
|
||||
- Don't resubmit when a submit reports `coalesced` — the work is already queued; monitor the matched job id instead
|
||||
- For subagent work, don't use `sessions_spawn` with `runtime: "subagent"` when Minions is available (use `gbrain agent run` instead)
|
||||
- Don't poll `get_job` in a tight loop (use `get_job_progress` for lightweight checks)
|
||||
- Don't run an operation expected to exceed ~2 minutes as a bare background shell — it dies with the session; route through the Durable execution ladder
|
||||
@@ -508,4 +521,5 @@ Total tokens so far: 4.3k
|
||||
- Replay a completed/failed job — `replay_job` (MCP)
|
||||
- Send sidechannel message — `send_job_message` (MCP)
|
||||
- Get structured progress — `get_job_progress` (MCP)
|
||||
- Queue stats — `gbrain jobs stats` (CLI; no MCP equivalent)
|
||||
- Queue stats — `get_job_stats` (MCP; admin scope over HTTP, same as the other
|
||||
jobs ops here — includes the wedged-queue silent-halt signal) or `gbrain jobs stats` (CLI)
|
||||
|
||||
@@ -177,8 +177,9 @@ Validate before sync:
|
||||
gbrain schema lint --with-db
|
||||
```
|
||||
|
||||
The `--with-db` flag opts into the 2 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`) that detect
|
||||
The `--with-db` flag opts into the 4 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`,
|
||||
`stored_type_is_alias`, `stored_type_undeclared`) that detect
|
||||
mis-declared types you'd otherwise discover only at runtime.
|
||||
|
||||
### Phase 5 — Sync (backfill existing pages with the new types)
|
||||
|
||||
@@ -119,9 +119,9 @@
|
||||
"migrations/v0.33.0.md": "11710cb11d6eb7dc3ea54b764e3c4a25f8679cf76590acd330f97bfa1c684945",
|
||||
"migrations/v0.33.3.0.md": "188a03ca86a97a9aa697cbbc83cc8ca37843fab24db2bd82f1383c400173d5bd",
|
||||
"migrations/v0.34.0.0.md": "d421c5ecff0765ac1de3592d3175734db7df52e8658ec101567779c7c56c2db2",
|
||||
"migrations/v0.35.0.0.md": "1f4083b6447ae694776f35b70ec6c259b03987cb35a8ee4376c8f51db6bb06ce",
|
||||
"migrations/v0.35.0.0.md": "a87d0f04f5d1d275c2f283c3736f0208bb9e24a540b0ffb79ec9d4c0de90e01f",
|
||||
"migrations/v0.35.7.0.md": "c6d4454bd39e2aa243b3b3d9bc72fe5a4fd25d097be7be2bb14b25604b5c2cc5",
|
||||
"migrations/v0.36.2.0.md": "becedef44dc377cf95c83ddc479c9ab19f15a984f716fc0b041ba7cde6886f14",
|
||||
"migrations/v0.36.2.0.md": "2b3b4cc0dc2e9611b0df9aea0a9cd4281433011c2d88433f768e762b28d56320",
|
||||
"migrations/v0.36.5.0.md": "a01a722202dfc3c799693596750c8bee611fe4dafe3cb662f6b4cd0b635cb429",
|
||||
"migrations/v0.40.3.0.md": "5f500f8c543c2b6f41778b0bd3beedada68f7284f7933ad8b769322b433a8fe9",
|
||||
"migrations/v0.40.5.md": "b9837d52a030517698dfb31c439f562cde60a1015ae488dab09be2c16ff182e5",
|
||||
@@ -133,7 +133,7 @@
|
||||
"migrations/v0.8.1.md": "fad7341cfb5e02545fb8a23221d12ab395fc3d8db15d1d8ee8a18844aea6563a",
|
||||
"migrations/v0.9.0.md": "773fab0a8d7f330576265a3f510c1f318f47789b6136c46d43e08121acbc20eb",
|
||||
"migrations/v0.9.1.md": "75761bad6c0ad37b69ec8197c6a678bb6a1484f9a76e4b70f2d1e86dc80102b3",
|
||||
"minion-orchestrator/SKILL.md": "5ddeff9bde80ef7fe4990c97220338ffc9ba0d2126eceaed7b4b6a3eb8b0fa18",
|
||||
"minion-orchestrator/SKILL.md": "0b6799dbe6bccc83984371545db0d1d6216d152578d035bb869471cca7ab9b69",
|
||||
"minion-orchestrator/routing-eval.jsonl": "501ed2e19cb16847ff8425219d246b7a774de1accd42cb28fd44edbb64204992",
|
||||
"perplexity-research/SKILL.md": "c25f5c471cbe3c6e0f975d8397e8382b00a85f8aa75302231d53c52855369e97",
|
||||
"perplexity-research/routing-eval.jsonl": "f1a40d87e710d5d2acd602a372d83f46c95da022b6e635228fffeaacb3bb2b27",
|
||||
@@ -149,7 +149,7 @@
|
||||
"research-compendium/routing-eval.jsonl": "7446cdcaf9c43fe2e20aaf129a705f13a7743f5f14455a7a21c663572def9078",
|
||||
"resolve-before-asking/SKILL.md": "1882c45b2e603bbb1e251d388cc2682270ee7eae99211d5a5322430f4667fb39",
|
||||
"resolve-before-asking/routing-eval.jsonl": "bac1bcf30337f5255ef4ce1a2a8a2b38d58ebcd576503c483190c79ec6e69489",
|
||||
"schema-author/SKILL.md": "4ac1c8fd08800f3728ec55cdc98e97a5aa618a26b753a0fb38c0df9624b66e06",
|
||||
"schema-author/SKILL.md": "1dd11a44dabcb7d57244be4cf5f4903feb9d146bcbb4363fc150daefc01d04ce",
|
||||
"schema-unify/SKILL.md": "e9ac84018d673d35f749a1f74380d635512308fa50951995a7cb339ab4c85fa6",
|
||||
"setup/SKILL.md": "7f11b70ed89d4bff87096aa7e7bb0d41191eb46682066f3b2cffa7a326b56330",
|
||||
"signal-detector/SKILL.md": "c85772f129b3a5b5b0edfa191e11b1048942e52b7472bbaea224e7188f8af75a",
|
||||
|
||||
+65
-17
@@ -65,6 +65,11 @@ export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'pglite-repair', 'upgr
|
||||
// v0.42.58 (#2035 class, caught by the handleCliOnly reachability sweep):
|
||||
// full handler at `case 'notability-eval'` but never dispatchable.
|
||||
'notability-eval',
|
||||
// #2035 class (wired the #3502 way): `case 'whoknows'` had a live handler
|
||||
// (runWhoknows: ranked table, per-factor explain, thin-client routing) that
|
||||
// was shadowed by find_experts' non-hidden cliHints. The op hint is now
|
||||
// hidden (ops/insights.ts); this entry makes the richer handler dispatch.
|
||||
'whoknows',
|
||||
// Agent-bootstrap family (ENG-2 three-touchpoint rule): `bootstrap` + `hook`
|
||||
// are ENGINE-FREE (dispatched in handleCliOnly before the connectEngine
|
||||
// terminator) and must NEVER enter THIN_CLIENT_REFUSED_COMMANDS. `sweep` is
|
||||
@@ -75,6 +80,8 @@ export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'pglite-repair', 'upgr
|
||||
// per-subcommand usage stays reachable.
|
||||
const CLI_ONLY_SELF_HELP = new Set([
|
||||
'upgrade', 'post-upgrade', 'check-update',
|
||||
// whoknows honours --help first (runWhoknows HELP block, whoknows.ts).
|
||||
'whoknows',
|
||||
// #3502 sweep: pages + bench print their own usage (pages.ts printHelp,
|
||||
// bench-publish.ts printHelp). Both were documented but undispatchable —
|
||||
// `pages` had a live handleCliOnly case but was missing from CLI_ONLY
|
||||
@@ -162,6 +169,9 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
// would hide both — `gbrain dream retriage --help` printed the one-line
|
||||
// dream stub instead of the retriage contract (outside-voice CX9).
|
||||
'dream',
|
||||
// ZE interim cleanup: the retired ze-switch shim ships truthful help
|
||||
// (sunset refusal + canonical migration command); the generic stub hid it.
|
||||
'ze-switch',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -170,7 +180,7 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
* answerable with no brain configured.
|
||||
*
|
||||
* Membership is behaviour, not taste: each entry is pinned by
|
||||
* test/cli-help-without-brain.test.ts, which runs the CLI with an empty
|
||||
* test/cli-help-without-brain.serial.test.ts, which runs the CLI with an empty
|
||||
* GBRAIN_HOME and requires exit 0 plus real help output.
|
||||
*/
|
||||
const SELF_HELP_WITHOUT_ENGINE: Record<string, () => Promise<(engine: never, args: string[]) => unknown>> = {
|
||||
@@ -187,6 +197,9 @@ const SELF_HELP_WITHOUT_ENGINE: Record<string, () => Promise<(engine: never, arg
|
||||
// runDream accepts BrainEngine | null; --help (and `retriage --help`) is
|
||||
// answered before any engine-bearing work per the dream.ts IRON RULE.
|
||||
dream: async () => (await import('./commands/dream.ts')).runDream as never,
|
||||
// The retired ze-switch shim answers --help engine-free (arg-order adapter
|
||||
// lives in ze-switch.ts because runZeSwitch takes (args, engine)).
|
||||
'ze-switch': async () => (await import('./commands/ze-switch.ts')).runZeSwitchSelfHelp as never,
|
||||
};
|
||||
|
||||
/** Returns true when the command's own help was printed. */
|
||||
@@ -391,6 +404,15 @@ async function main() {
|
||||
if (command === 'search' && ['modes', 'stats', 'tune', 'diagnose'].includes(subArgs[0] ?? '')) {
|
||||
const { withTimeout, OperationTimeoutError } = await import('./core/timeout.ts');
|
||||
const isDiagnose = subArgs[0] === 'diagnose';
|
||||
// Gap-closure wave [OV6]: thin clients route the read-only dashboard
|
||||
// forms via search_modes/search_stats/search_tune instead of fabricating
|
||||
// a scratch PGLite; --reset/--apply/diagnose fall through to the refusal.
|
||||
const cfgSearch = loadConfig();
|
||||
if (isThinClient(cfgSearch)) {
|
||||
const { routeThinClientCommand } = await import('./commands/thin-client-routing.ts');
|
||||
if (await routeThinClientCommand(cfgSearch!, 'search', subArgs)) return;
|
||||
refuseThinClient('search', cfgSearch!.remote_mcp!.mcp_url);
|
||||
}
|
||||
const label = 'gbrain search';
|
||||
// diagnose runs real retrieval (keyword + vector + hybrid) so it gets a
|
||||
// longer deadline than the read-only dashboard.
|
||||
@@ -1660,7 +1682,7 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
orphans: "orphans needs the host's brain. Run on the host or use the `find_orphans` MCP tool from your agent.",
|
||||
transcripts: 'transcripts is server-private (raw chat exports stay on the host). Read transcripts on the host machine.',
|
||||
storage: 'storage operates on the local repo on disk. Run on the host.',
|
||||
takes: 'takes mutate subcommands edit local .md files; routing the read subcommands lands in v0.31.x. For now: use `takes_list` and `takes_search` MCP tools from your agent, or run on the host.',
|
||||
takes: 'takes list/search/scorecard/calibration + add/update/resolve/supersede route to the brain host automatically (takes_* MCP ops). This subcommand (extract/revisit) is host-bound: run it on the host machine.',
|
||||
sources: 'sources commands manage local DB + config rows. Per-subcommand thin-client routing lands in v0.31.x. For now: use `sources_list` / `sources_status` MCP tools, or run on the host.',
|
||||
sweep: 'sweep runs the serve-resident maintenance passes against the LOCAL engine. Run it on the host (the serve process also runs it automatically).',
|
||||
// v0.32 audit additions
|
||||
@@ -1673,7 +1695,12 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
'code-callees': '`code-callees` has no MCP op yet. Run on the host.',
|
||||
// scratch-DB audit additions
|
||||
config: "config reads/writes the host brain's config plane. Edit the host's .gbrain/config.json (file-plane keys) or run on the host with GBRAIN_HOME set.",
|
||||
jobs: '`jobs list` and `jobs get <id>` are thin-client routable; this subcommand runs against the host queue. Use the submit_job / list_jobs / get_job MCP tools from your agent, or run on the host with GBRAIN_HOME set.',
|
||||
jobs: '`jobs list`, `jobs get <id>`, and `jobs stats` are thin-client routable; this subcommand runs against the host queue. Use the submit_job / list_jobs / get_job / get_job_stats MCP tools from your agent, or run on the host with GBRAIN_HOME set.',
|
||||
// Gap-closure wave [OV6]: routable subcommands are intercepted before this
|
||||
// hint fires — these fire only for the host-bound remainder.
|
||||
search: '`search modes|stats|tune` route to the brain host automatically (search_modes / search_stats / search_tune MCP ops). The modes reset form, modes with the source flag (the reset dry-run), and tune apply mutate or preview host config, and `diagnose` runs live retrieval — run those on the host.',
|
||||
cache: '`cache stats` routes to the brain host automatically (cache_stats MCP op). clear/prune mutate the host cache — run those on the host.',
|
||||
quarantine: '`quarantine list` routes to the brain host automatically (quarantine_list MCP op). scan/clear are host-bound (bulk re-import; the clear trust decision) — run those on the host.',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1701,9 +1728,14 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
// hint instead of letting them fail later inside connectEngine or
|
||||
// mid-handler. v0.31.1 routes through `refuseThinClient` so every
|
||||
// refusal carries an actionable next-step hint (CDX-5 cherry-pick A).
|
||||
if (THIN_CLIENT_REFUSED_COMMANDS.has(command)) {
|
||||
// Gap-closure wave [OV6]: takes/cache/quarantine first try the
|
||||
// per-subcommand MCP routing (engine-free); unhandled subcommands fall
|
||||
// through to the refusal.
|
||||
if (THIN_CLIENT_REFUSED_COMMANDS.has(command) || command === 'cache' || command === 'quarantine') {
|
||||
const cfg = loadConfig();
|
||||
if (isThinClient(cfg)) {
|
||||
const { routeThinClientCommand } = await import('./commands/thin-client-routing.ts');
|
||||
if (await routeThinClientCommand(cfg!, command, args)) return;
|
||||
refuseThinClient(command, cfg!.remote_mcp!.mcp_url);
|
||||
}
|
||||
}
|
||||
@@ -2009,10 +2041,31 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
|
||||
if (command === 'ze-switch') {
|
||||
// v0.36.0.0 — manual ZE-default switch lever. Owns its own engine lifecycle
|
||||
// to mirror the doctor pattern.
|
||||
// Retired refusal/redirect shim. Only --undo reads the brain (one config
|
||||
// row); every other invocation must refuse EVEN ON an unconfigured
|
||||
// machine — connecting unconditionally turned the refusal into
|
||||
// "No brain configured" and starved --json callers of the envelope.
|
||||
const { runZeSwitch } = await import('./commands/ze-switch.ts');
|
||||
const eng = await connectEngine();
|
||||
if (!args.includes('--undo')) {
|
||||
await runZeSwitch(args, null);
|
||||
return;
|
||||
}
|
||||
// --undo reads one config row. An unconfigured machine (or a failed
|
||||
// connect) must still get the shim's truthful --json refusal envelope —
|
||||
// connectEngine would print plain "No brain configured" and exit before
|
||||
// the shim ran, so pre-check the config and degrade to a null engine
|
||||
// (the shim words that as a read failure).
|
||||
if (!loadConfig()) {
|
||||
await runZeSwitch(args, null);
|
||||
return;
|
||||
}
|
||||
let eng: BrainEngine | null = null;
|
||||
try {
|
||||
eng = await connectEngine();
|
||||
} catch {
|
||||
await runZeSwitch(args, null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await runZeSwitch(args, eng);
|
||||
} finally {
|
||||
@@ -2348,6 +2401,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runJobs(null, args);
|
||||
return;
|
||||
}
|
||||
if (jobsSub === 'stats') {
|
||||
// Gap-closure wave [OV6]: queue health routes via get_job_stats.
|
||||
const { routeThinClientCommand } = await import('./commands/thin-client-routing.ts');
|
||||
if (await routeThinClientCommand(cfgJobs!, 'jobs', args)) return;
|
||||
}
|
||||
refuseThinClient('jobs', cfgJobs!.remote_mcp!.mcp_url);
|
||||
}
|
||||
}
|
||||
@@ -2701,12 +2759,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runModels(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'search': {
|
||||
// v0.32.3 search-lite — `gbrain search modes/stats/tune`.
|
||||
const { runSearch } = await import('./commands/search.ts');
|
||||
await runSearch(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'takes': {
|
||||
const { runTakes } = await import('./commands/takes.ts');
|
||||
await runTakes(engine, args);
|
||||
@@ -3124,10 +3176,6 @@ export function printOpHelp(op: Operation, invokedName?: string) {
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
// Gather shared operations grouped by category
|
||||
const cliNames = Array.from(cliOps.entries())
|
||||
.map(([name, op]) => ({ name, desc: op.description }));
|
||||
|
||||
console.log(`gbrain ${VERSION} -- personal knowledge brain
|
||||
|
||||
USAGE
|
||||
|
||||
+31
-5
@@ -16,6 +16,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../core/minions/admission.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../core/minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, SubagentHandlerData, AggregatorHandlerData } from '../core/minions/types.ts';
|
||||
import { resolveSourceId, ALL_SOURCES } from '../core/source-resolver.ts';
|
||||
@@ -313,7 +314,13 @@ export async function runAgentRun(engine: BrainEngine, args: string[]): Promise<
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
|
||||
process.stderr.write(`submitted: job ${job.id} (subagent)\n`);
|
||||
// Honest-dispatch at the interactive surface (codex re-review): a
|
||||
// param-coalesced submit returns an EXISTING waiting job — printing
|
||||
// 'submitted' would tell the operator a new run was queued when it wasn't.
|
||||
process.stderr.write(job.coalesced === true
|
||||
? `coalesced: identical params matched existing waiting job ${job.id} (subagent). ` +
|
||||
`Vary the prompt/params or pass a fresh idempotency key for an independent run.\n`
|
||||
: `submitted: job ${job.id} (subagent)\n`);
|
||||
|
||||
if (flags.detach || !flags.follow) {
|
||||
process.stdout.write(String(job.id) + '\n');
|
||||
@@ -361,7 +368,9 @@ async function runFanout(engine: BrainEngine, queue: MinionQueue, flags: RunFlag
|
||||
const job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
process.stderr.write(`submitted: job ${job.id} (single-entry manifest short-circuit)\n`);
|
||||
process.stderr.write(job.coalesced === true
|
||||
? `coalesced: identical params matched existing waiting job ${job.id} (single-entry manifest short-circuit).\n`
|
||||
: `submitted: job ${job.id} (single-entry manifest short-circuit)\n`);
|
||||
if (flags.detach || !flags.follow) { process.stdout.write(`${job.id}\n`); return; }
|
||||
await followJob(engine, queue, job.id, flags.timeoutMs);
|
||||
return;
|
||||
@@ -394,9 +403,26 @@ async function runFanout(engine: BrainEngine, queue: MinionQueue, flags: RunFlag
|
||||
max_stalled: 3,
|
||||
};
|
||||
if (flags.timeoutMs) submitOpts.timeout_ms = flags.timeoutMs;
|
||||
const child = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
let child;
|
||||
try {
|
||||
child = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
} catch (e) {
|
||||
// Admission quota mid-fanout: a partial tree (some children submitted,
|
||||
// children_ids never written) would leave the aggregator torn — cancel
|
||||
// the WHOLE tree (cascades to already-submitted children) and surface
|
||||
// the quota message. All-or-nothing beats a wedged aggregator.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
await queue.cancelJob(aggregator.id).catch(() => {});
|
||||
console.error(
|
||||
`fanout aborted at child ${childIds.length + 1}/${manifest.length}: ${e.message}\n` +
|
||||
`Aggregator ${aggregator.id} and its ${childIds.length} submitted child(ren) were cancelled.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
childIds.push(child.id);
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,11 @@ import { VERSION } from '../version.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { loadCompletedMigrations, appendCompletedMigration, type CompletedMigrationEntry } from '../core/preferences.ts';
|
||||
import { migrations, compareVersions, type Migration, type OrchestratorOpts } from './migrations/index.ts';
|
||||
|
||||
/** Bug 3 — max consecutive partials before we wedge a migration. */
|
||||
const MAX_CONSECUTIVE_PARTIALS = 3;
|
||||
import {
|
||||
indexCompletedEntries,
|
||||
statusForVersion as ledgerStatusForVersion,
|
||||
MAX_CONSECUTIVE_PARTIALS,
|
||||
} from '../core/migration-ledger.ts';
|
||||
|
||||
interface ApplyMigrationsArgs {
|
||||
list: boolean;
|
||||
@@ -117,53 +119,18 @@ interface CompletedIndex {
|
||||
byVersion: Map<string, CompletedMigrationEntry[]>;
|
||||
}
|
||||
|
||||
// Ledger status logic moved to src/core/migration-ledger.ts (shared with the
|
||||
// get_health op's migrations block, TODOS:4063) — same semantics, same Bug 3
|
||||
// "complete wins / trailing retry overrides / consecutive-partial cap" rules.
|
||||
function indexCompleted(entries: CompletedMigrationEntry[]): CompletedIndex {
|
||||
const byVersion = new Map<string, CompletedMigrationEntry[]>();
|
||||
for (const e of entries) {
|
||||
const list = byVersion.get(e.version) ?? [];
|
||||
list.push(e);
|
||||
byVersion.set(e.version, list);
|
||||
}
|
||||
return byVersion.size > 0
|
||||
? { byVersion }
|
||||
: { byVersion: new Map() };
|
||||
return { byVersion: indexCompletedEntries(entries) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the resolved status for a migration based on its entries.
|
||||
*
|
||||
* Semantics (Bug 3 — keep "complete wins" safety):
|
||||
* - If the latest entry is `retry`, the version is pending. This is the
|
||||
* explicit escape hatch written by `--force-retry`, and it overrides an
|
||||
* earlier `complete` entry without hand-editing the ledger.
|
||||
* - Otherwise, if any entry is `complete`, the version is complete.
|
||||
* - Otherwise, if any entry is `partial`, the version is partial.
|
||||
* - Otherwise, pending.
|
||||
*
|
||||
* `complete` never regresses accidentally. A later `partial` append cannot
|
||||
* undo a completed migration; only a trailing, explicit `retry` marker can.
|
||||
*/
|
||||
function statusForVersion(
|
||||
version: string,
|
||||
idx: CompletedIndex,
|
||||
): 'complete' | 'partial' | 'pending' | 'wedged' {
|
||||
const entries = idx.byVersion.get(version) ?? [];
|
||||
if (entries.length === 0) return 'pending';
|
||||
const latest = entries[entries.length - 1];
|
||||
if (latest.status === 'retry') return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
// Bug 3 attempt cap — count consecutive partials from the end (stopping
|
||||
// at any 'retry' or 'complete'). If we hit MAX_CONSECUTIVE_PARTIALS,
|
||||
// the migration is wedged and needs explicit --force-retry to try again.
|
||||
let consecutive = 0;
|
||||
for (let i = entries.length - 1; i >= 0; i--) {
|
||||
const e = entries[i];
|
||||
if (e.status === 'partial') consecutive++;
|
||||
else break;
|
||||
}
|
||||
if (consecutive >= MAX_CONSECUTIVE_PARTIALS) return 'wedged';
|
||||
if (entries.some(e => e.status === 'partial')) return 'partial';
|
||||
return 'pending';
|
||||
return ledgerStatusForVersion(version, idx.byVersion);
|
||||
}
|
||||
|
||||
interface Plan {
|
||||
|
||||
+155
-37
@@ -10,13 +10,16 @@
|
||||
* gbrain check-backlinks fix --dry-run # preview fixes
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync, statSync, lstatSync, existsSync } from 'fs';
|
||||
import { readFileSync, readdirSync, statSync, lstatSync, existsSync } from 'fs';
|
||||
import { join, relative, basename } from 'path';
|
||||
import { extractEntityRefs as canonicalExtractEntityRefs } from '../core/link-extraction.ts';
|
||||
import { createProgress, startHeartbeat } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { parseMarkdown, frontmatterBodyOffset } from '../core/markdown.ts';
|
||||
import { atomicWriteFileSync } from '../core/atomic-write.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
|
||||
interface BacklinkGap {
|
||||
export interface BacklinkGap {
|
||||
/** The page that mentions the entity */
|
||||
sourcePage: string;
|
||||
/** The entity page that's missing the back-link */
|
||||
@@ -132,10 +135,77 @@ export function findBacklinkGaps(brainDir: string): BacklinkGap[] {
|
||||
return gaps;
|
||||
}
|
||||
|
||||
/** Fix back-link gaps by appending timeline entries to target pages */
|
||||
export function fixBacklinkGaps(brainDir: string, gaps: BacklinkGap[], dryRun: boolean = false): number {
|
||||
/** Per-run outcome of the fixer: entries inserted + per-file skip reasons. */
|
||||
export interface BacklinkFixOutcome {
|
||||
fixed: number;
|
||||
skipped: Array<{ page: string; reason: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation codes that make a file UNSAFE to edit: the fence/YAML itself is
|
||||
* broken (or the offset math would be unreliable), so any body insertion could
|
||||
* worsen the damage. Deliberately NOT in this set: MISSING_OPEN (a legacy page
|
||||
* with no frontmatter at all has no fence to corrupt — the whole file is body
|
||||
* and stays fixable) and the content-quality lint codes (NESTED_QUOTES,
|
||||
* NON_STRING_FIELD, EMPTY_FRONTMATTER, SLUG_MISMATCH) whose presence doesn't
|
||||
* affect where the body starts.
|
||||
*/
|
||||
const EDIT_BLOCKING_CODES = new Set(['YAML_PARSE', 'MISSING_CLOSE', 'NULL_BYTES']);
|
||||
|
||||
function firstEditBlockingError(content: string, filePath: string): string | null {
|
||||
const parsed = parseMarkdown(content, filePath, { validate: true });
|
||||
const blocking = (parsed.errors ?? []).find(e => EDIT_BLOCKING_CODES.has(e.code));
|
||||
return blocking ? `${blocking.code}: ${blocking.message}` : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a timeline entry into the body of `content`, never touching bytes
|
||||
* before `bodyStart`. The `## Timeline` heading is matched only as a real
|
||||
* heading line at/after bodyStart (CRLF-tolerant), so a `## Timeline` string
|
||||
* inside YAML frontmatter, a `### Timeline` sub-heading, or a
|
||||
* `## Timeline (2026)` variant never anchors the insertion. With multiple real
|
||||
* headings, the FIRST one wins deterministically (post-validation guards the
|
||||
* result either way). Exported for direct unit tests.
|
||||
*/
|
||||
export function insertTimelineEntry(content: string, bodyStart: number, entry: string): string {
|
||||
const bodySlice = content.slice(bodyStart);
|
||||
const headingMatch = /^## Timeline[ \t]*\r?$/m.exec(bodySlice);
|
||||
|
||||
if (!headingMatch) {
|
||||
// No real Timeline heading in the body — append a fresh section.
|
||||
return content.trimEnd() + '\n\n## Timeline\n\n' + entry + '\n';
|
||||
}
|
||||
|
||||
const headingAbs = bodyStart + headingMatch.index;
|
||||
const headingLineEnd = content.indexOf('\n', headingAbs);
|
||||
const sectionStart = headingLineEnd === -1 ? content.length : headingLineEnd + 1;
|
||||
|
||||
const nextHeading = /^## /m.exec(content.slice(sectionStart));
|
||||
if (nextHeading) {
|
||||
const insertAt = sectionStart + nextHeading.index;
|
||||
return content.slice(0, insertAt) + entry + '\n' + content.slice(insertAt);
|
||||
}
|
||||
return content.trimEnd() + '\n' + entry + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix back-link gaps by inserting timeline entries into target pages.
|
||||
*
|
||||
* Safety pipeline per target file (each failure isolates to that file and is
|
||||
* reported in `skipped` — one bad page can't kill the batch or corrupt itself):
|
||||
* lock (withPageLock) → read → pre-validate (skip if the fence/YAML is
|
||||
* already broken) → insert after the frontmatter-safe body offset →
|
||||
* post-validate the candidate → atomic write (tmp+fsync+rename) that
|
||||
* re-validates the on-disk bytes before the rename.
|
||||
*/
|
||||
export async function fixBacklinkGaps(
|
||||
brainDir: string,
|
||||
gaps: BacklinkGap[],
|
||||
dryRun: boolean = false,
|
||||
opts?: { lockRoot?: string },
|
||||
): Promise<BacklinkFixOutcome> {
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
let fixed = 0;
|
||||
const outcome: BacklinkFixOutcome = { fixed: 0, skipped: [] };
|
||||
|
||||
// Group gaps by target page to batch writes
|
||||
const byTarget = new Map<string, BacklinkGap[]>();
|
||||
@@ -149,42 +219,62 @@ export function fixBacklinkGaps(brainDir: string, gaps: BacklinkGap[], dryRun: b
|
||||
const targetPath = join(brainDir, targetPage);
|
||||
if (!existsSync(targetPath)) continue;
|
||||
|
||||
let content = readFileSync(targetPath, 'utf-8');
|
||||
const lockKey = targetPage.replace(/\.md$/, '');
|
||||
try {
|
||||
await withPageLock(lockKey, async () => {
|
||||
let content = readFileSync(targetPath, 'utf-8');
|
||||
|
||||
for (const gap of targetGaps) {
|
||||
// Compute relative path from target to source
|
||||
const targetDir = targetPage.split('/').slice(0, -1);
|
||||
const sourceDir = gap.sourcePage.split('/');
|
||||
const depth = targetDir.length;
|
||||
const relPrefix = '../'.repeat(depth);
|
||||
const relPath = relPrefix + gap.sourcePage;
|
||||
|
||||
const entry = buildBacklinkEntry(gap.sourceTitle, relPath, today);
|
||||
|
||||
// Insert into Timeline section
|
||||
if (content.includes('## Timeline')) {
|
||||
const parts = content.split('## Timeline');
|
||||
const afterTimeline = parts[1];
|
||||
const nextSection = afterTimeline.match(/\n## /);
|
||||
if (nextSection) {
|
||||
const insertIdx = parts[0].length + '## Timeline'.length + nextSection.index!;
|
||||
content = content.slice(0, insertIdx) + '\n' + entry + content.slice(insertIdx);
|
||||
} else {
|
||||
content = content.trimEnd() + '\n' + entry + '\n';
|
||||
const preError = firstEditBlockingError(content, targetPath);
|
||||
if (preError) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: `pre-existing invalid frontmatter (${preError}) — file left untouched`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Add Timeline section
|
||||
content = content.trimEnd() + '\n\n## Timeline\n\n' + entry + '\n';
|
||||
}
|
||||
fixed++;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
writeFileSync(targetPath, content);
|
||||
const bodyStart = frontmatterBodyOffset(content);
|
||||
let inserted = 0;
|
||||
for (const gap of targetGaps) {
|
||||
// Compute relative path from target to source
|
||||
const targetDir = targetPage.split('/').slice(0, -1);
|
||||
const depth = targetDir.length;
|
||||
const relPrefix = '../'.repeat(depth);
|
||||
const relPath = relPrefix + gap.sourcePage;
|
||||
|
||||
const entry = buildBacklinkEntry(gap.sourceTitle, relPath, today);
|
||||
content = insertTimelineEntry(content, bodyStart, entry);
|
||||
inserted++;
|
||||
}
|
||||
|
||||
const postError = firstEditBlockingError(content, targetPath);
|
||||
if (postError) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: `edit would invalidate page (${postError}) — aborted, file left untouched`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
atomicWriteFileSync(targetPath, content, {
|
||||
verify: (onDisk) => {
|
||||
const diskError = firstEditBlockingError(onDisk, targetPath);
|
||||
if (diskError) throw new Error(`on-disk validation failed (${diskError})`);
|
||||
},
|
||||
});
|
||||
}
|
||||
outcome.fixed += inserted;
|
||||
}, { timeoutMs: 10_000, lockRoot: opts?.lockRoot });
|
||||
} catch (e) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return fixed;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
export interface BacklinksOpts {
|
||||
@@ -199,6 +289,9 @@ export interface BacklinksResult {
|
||||
fixed: number;
|
||||
pages_affected: number;
|
||||
dryRun: boolean;
|
||||
/** Pages the fixer refused to touch (invalid frontmatter, lock/write errors). */
|
||||
skipped_invalid?: number;
|
||||
skipped_pages?: Array<{ page: string; reason: string }>;
|
||||
}
|
||||
|
||||
export interface ParsedBacklinksArgs {
|
||||
@@ -263,8 +356,27 @@ export async function runBacklinksCore(opts: BacklinksOpts): Promise<BacklinksRe
|
||||
const pagesAffected = new Set(gaps.map(g => g.targetPage)).size;
|
||||
|
||||
if (opts.action === 'fix' && gaps.length > 0) {
|
||||
const fixed = fixBacklinkGaps(opts.dir, gaps, !!opts.dryRun);
|
||||
return { action: 'fix', gaps_found: gaps.length, fixed, pages_affected: pagesAffected, dryRun: !!opts.dryRun };
|
||||
// Locks + per-file validation make the fix loop slower than the naive
|
||||
// writer it replaced — run it under its own phase with a heartbeat so
|
||||
// agents see forward progress (the scan phase above already finished).
|
||||
progress.start('backlinks.fix');
|
||||
const fixHb = startHeartbeat(progress, 'applying back-link fixes…');
|
||||
let fixOutcome: BacklinkFixOutcome;
|
||||
try {
|
||||
fixOutcome = await fixBacklinkGaps(opts.dir, gaps, !!opts.dryRun);
|
||||
} finally {
|
||||
fixHb();
|
||||
progress.finish();
|
||||
}
|
||||
return {
|
||||
action: 'fix',
|
||||
gaps_found: gaps.length,
|
||||
fixed: fixOutcome.fixed,
|
||||
pages_affected: pagesAffected,
|
||||
dryRun: !!opts.dryRun,
|
||||
skipped_invalid: fixOutcome.skipped.length,
|
||||
skipped_pages: fixOutcome.skipped,
|
||||
};
|
||||
}
|
||||
return { action: opts.action, gaps_found: gaps.length, fixed: 0, pages_affected: pagesAffected, dryRun: !!opts.dryRun };
|
||||
}
|
||||
@@ -310,6 +422,12 @@ export async function runBacklinks(args: string[]) {
|
||||
} else {
|
||||
const label = result.dryRun ? '(dry run) ' : '';
|
||||
console.log(`${label}Fixed ${result.fixed} missing back-link(s) across ${result.pages_affected} page(s).`);
|
||||
if (result.skipped_pages && result.skipped_pages.length > 0) {
|
||||
console.log(`\nSkipped ${result.skipped_pages.length} page(s):`);
|
||||
for (const s of result.skipped_pages) {
|
||||
console.log(` ${s.page}: ${s.reason}`);
|
||||
}
|
||||
}
|
||||
if (result.dryRun) {
|
||||
console.log('\nRe-run without --dry-run to apply.');
|
||||
}
|
||||
|
||||
+12
-163
@@ -31,7 +31,6 @@
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import matter from 'gray-matter';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult, RemoteMcpError } from '../core/mcp-client.ts';
|
||||
@@ -39,6 +38,18 @@ import { computeContentHash } from '../core/ingestion/types.ts';
|
||||
import { operations } from '../core/operations.ts';
|
||||
import type { OperationContext } from '../core/operations.ts';
|
||||
import { resolveSourceWithTier } from '../core/source-resolver.ts';
|
||||
// Pure content helpers moved to core (shared with the capture MCP op — the
|
||||
// core module also breaks the capture.ts→operations.ts static import cycle).
|
||||
// Re-exported below so existing importers/tests keep their entry point.
|
||||
import {
|
||||
defaultSlug,
|
||||
detectBinaryNullByte,
|
||||
normalizeForHash,
|
||||
deriveTitle,
|
||||
mergeCaptureFrontmatter,
|
||||
} from '../core/capture-content.ts';
|
||||
|
||||
export { detectBinaryNullByte, normalizeForHash, mergeCaptureFrontmatter } from '../core/capture-content.ts';
|
||||
|
||||
interface RunOpts {
|
||||
content?: string;
|
||||
@@ -144,45 +155,6 @@ Examples:
|
||||
JOB=$(gbrain capture "..." --quiet)
|
||||
`;
|
||||
|
||||
// v0.42.x — Life Chronicle (#2390): route the default slug prefix by type so
|
||||
// `gbrain capture --type diary` lands under life/diary/ and `--type event`
|
||||
// under life/events/ (matching the chronicle path-prefix inference). Everything
|
||||
// else keeps the inbox/ default.
|
||||
function slugPrefixForType(type?: string): string {
|
||||
if (type === 'diary') return 'life/diary';
|
||||
if (type === 'event') return 'life/events';
|
||||
return 'inbox';
|
||||
}
|
||||
function defaultSlug(content: string, now: Date = new Date(), type?: string): string {
|
||||
const y = now.getUTCFullYear();
|
||||
const m = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
const d = String(now.getUTCDate()).padStart(2, '0');
|
||||
const hashPrefix = computeContentHash(content).slice(0, 8);
|
||||
return `${slugPrefixForType(type)}/${y}-${m}-${d}-${hashPrefix}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 CV10 — binary file guard. Scans the first 8KB of `buf` for a
|
||||
* NUL byte (0x00). Real text files (including UTF-8 with multi-byte CJK,
|
||||
* emoji, BOM) never contain a NUL byte at any position — text encoding
|
||||
* uses non-zero continuation bytes. NUL appears in binary formats:
|
||||
* executables, archives, compressed images, PDFs (after the magic-byte
|
||||
* header), most office documents. Single-pass scan; constant memory.
|
||||
*
|
||||
* Returns the 0-indexed byte offset of the first NUL, or -1 if clean.
|
||||
* Caller decides the error shape (message vs JSON envelope).
|
||||
*
|
||||
* Known limit: a PNG-without-NUL-in-first-8KB slips through. v0.39
|
||||
* magic-byte allowlist (per CV10-B + TODOS.md) closes this hole. The
|
||||
* 8KB ceiling bounds the scan cost to ~microseconds even on huge files.
|
||||
*/
|
||||
export function detectBinaryNullByte(buf: Buffer): number {
|
||||
const limit = Math.min(buf.length, 8 * 1024);
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (buf[i] === 0) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
async function readStdinBuffer(): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
@@ -192,20 +164,6 @@ async function readStdinBuffer(): Promise<Buffer> {
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 CV9 — normalize content for content_hash so identical text
|
||||
* produces identical hashes regardless of leading/trailing whitespace,
|
||||
* line-ending style (CRLF vs LF), or Unicode normalization form. The
|
||||
* STORED body is preserved as-is (CRLF stays CRLF, BOM stays BOM).
|
||||
*
|
||||
* Two concerns, two transforms — the hash gets aggressive normalization
|
||||
* for dedup correctness; the stored body keeps user bytes for round-trip
|
||||
* fidelity. CQ2's CRLF/BOM preservation tests rely on this split.
|
||||
*/
|
||||
export function normalizeForHash(s: string): string {
|
||||
// Strip BOM, normalize line endings to LF, trim, NFKC for Unicode-stable hash.
|
||||
return s.replace(/^/, '').replace(/\r\n/g, '\n').trim().normalize('NFKC');
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 A2 + CV6 — detect Postgres FK violation on the sources table
|
||||
@@ -231,115 +189,6 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef
|
||||
return `source '${sourceId}' is not registered. Register it first:\n gbrain sources add ${sourceId} --path <path>\n\nList registered sources:\n gbrain sources list`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a title from the first non-empty, non-`---` line of the body,
|
||||
* stripping leading markdown heading marks, capped at 80 chars. Truncation
|
||||
* is codepoint-aware (never splits an astral surrogate pair) and appends an
|
||||
* ellipsis so a cut title is visibly cut.
|
||||
* Falls back to 'Capture' when no usable line exists.
|
||||
*/
|
||||
function deriveTitle(rawBody: string): string {
|
||||
const firstLine = rawBody
|
||||
.split('\n')
|
||||
.find((l) => l.trim().length > 0 && l.trim() !== '---') ?? '';
|
||||
const stripped = firstLine.replace(/^#+\s*/, '');
|
||||
const cps = [...stripped];
|
||||
return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture';
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 (BUG-1): merge capture's auto-stamped fields with any existing
|
||||
* frontmatter in `rawBody`, rather than always prepending a second
|
||||
* frontmatter block. The pre-fix code stamped its own `---` block on top
|
||||
* of files that already had frontmatter, producing `title: '---'` (the
|
||||
* file's opening delimiter became the outer title) and two consecutive
|
||||
* frontmatter blocks the parser interpreted as the outer block + a body
|
||||
* starting with a horizontal rule.
|
||||
*
|
||||
* Precedence rules (user-wins by default):
|
||||
* - `type`: opts.type (CLI flag) > userFm.type > 'note'
|
||||
* - `title`: userFm.title > derived-from-body
|
||||
* - `captured_via`: userFm.captured_via > opts.source > 'capture-cli'
|
||||
* (CV3/Phase 3c will narrow this to always 'capture-cli';
|
||||
* for Phase 2a we preserve current semantics)
|
||||
* - `captured_at`: userFm.captured_at > now (user can pre-stamp for retroactive
|
||||
* captures; see CQ2 test case 4)
|
||||
* - Any other user-declared keys (description, tags, slug, etc.) pass through verbatim.
|
||||
*
|
||||
* For files WITHOUT existing frontmatter, preserves the original behavior:
|
||||
* stamps a fresh frontmatter block, and if the body doesn't already look
|
||||
* like markdown (no `#` heading), wraps it under a `# {title}` heading.
|
||||
*/
|
||||
// v0.42.x — Life Chronicle (#2390): assemble the `event:` frontmatter block
|
||||
// from the --who/--what/--where/--kind/--depth flags (only for --type event).
|
||||
// Returns undefined when no event flags are set so non-event captures are
|
||||
// untouched.
|
||||
function buildEventBlock(opts: RunOpts): Record<string, unknown> | undefined {
|
||||
if (opts.type !== 'event') return undefined;
|
||||
const who = opts.who ? opts.who.split(',').map((s) => s.trim()).filter(Boolean) : [];
|
||||
const block: Record<string, unknown> = {};
|
||||
if (opts.what) block.what = opts.what;
|
||||
if (who.length) block.who = who;
|
||||
if (opts.where) block.where = opts.where;
|
||||
if (opts.kind) block.kind = opts.kind;
|
||||
if (opts.depth) block.depth = opts.depth;
|
||||
return Object.keys(block).length ? block : undefined;
|
||||
}
|
||||
|
||||
export function mergeCaptureFrontmatter(rawBody: string, opts: RunOpts): string {
|
||||
const nowIso = new Date().toISOString();
|
||||
// Detect frontmatter: leading `---\n` or `---\r\n`, tolerating leading BOM/whitespace.
|
||||
// We do NOT use the more permissive `startsWith('---')` because a body that opens
|
||||
// with a horizontal-rule like `--- separator ---` would false-positive.
|
||||
const trimmedStart = rawBody.replace(/^/, '');
|
||||
const hasFrontmatter = /^---\r?\n/.test(trimmedStart);
|
||||
|
||||
if (!hasFrontmatter) {
|
||||
// No existing frontmatter: stamp a fresh block and (if body lacks markdown
|
||||
// structure) wrap under a derived heading.
|
||||
const title = deriveTitle(rawBody);
|
||||
const fm: Record<string, unknown> = {
|
||||
type: opts.type ?? 'note',
|
||||
title,
|
||||
captured_via: opts.source ?? 'capture-cli',
|
||||
captured_at: nowIso,
|
||||
};
|
||||
const ev = buildEventBlock(opts);
|
||||
if (ev) fm.event = ev;
|
||||
const looksMarkdown = /^#{1,6}\s/.test(rawBody.trimStart());
|
||||
const body = looksMarkdown ? rawBody : `# ${title}\n\n${rawBody}`;
|
||||
return matter.stringify(body, fm);
|
||||
}
|
||||
|
||||
// Existing frontmatter: parse, merge user-wins, re-emit as a SINGLE block.
|
||||
let parsed: matter.GrayMatterFile<string>;
|
||||
try {
|
||||
parsed = matter(rawBody);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`malformed frontmatter in capture input: ${e instanceof Error ? e.message : String(e)}`,
|
||||
);
|
||||
}
|
||||
const userFm = (parsed.data ?? {}) as Record<string, unknown>;
|
||||
const merged: Record<string, unknown> = {
|
||||
// Spread user's declared keys first so 'description', 'tags', etc. pass through.
|
||||
...userFm,
|
||||
// Then apply auto-fields with the precedence rules above. The explicit
|
||||
// assignment AFTER the spread is intentional: it lets us implement the
|
||||
// mixed precedence (CLI flag wins for `type`; user wins for `title`/
|
||||
// `captured_via`/`captured_at`) in one expression per key.
|
||||
type: opts.type ?? userFm.type ?? 'note',
|
||||
title: userFm.title ?? deriveTitle(parsed.content),
|
||||
captured_via: userFm.captured_via ?? opts.source ?? 'capture-cli',
|
||||
captured_at: userFm.captured_at ?? nowIso,
|
||||
};
|
||||
// v0.42.x — merge the event block (user-declared keys win per-key).
|
||||
const ev = buildEventBlock(opts);
|
||||
if (ev || userFm.event) {
|
||||
merged.event = { ...(ev ?? {}), ...((userFm.event as Record<string, unknown>) ?? {}) };
|
||||
}
|
||||
return matter.stringify(parsed.content, merged);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the put_page content (frontmatter + body). The user's --type and
|
||||
|
||||
@@ -117,15 +117,17 @@ export const AGENT_SPECS: Record<AgentId, AgentSpec> = {
|
||||
export const AGENT_IDS: AgentId[] = ['claude-code', 'codex', 'opencode', 'perplexity', 'generic'];
|
||||
|
||||
// The named tools MUST be real MCP-exposed ops (verified by the round-trip
|
||||
// E2E). `capture` is intentionally absent: it's a CLI-only convenience wrapper,
|
||||
// not an MCP tool — the agent writes over MCP with `put_page`.
|
||||
// E2E). `capture` earned its slot in the CLI→MCP gap-closure wave (D2A):
|
||||
// it is a starter-surface op now — prefer it for quick notes (auto-slug +
|
||||
// dedupe), `put_page` for full-control writes.
|
||||
export const LEARN_INSTRUCTION =
|
||||
'Once connected, call the `get_brain_identity` tool (whose brain this is), then ' +
|
||||
'`list_skills` (everything it can do; if it errors, the host has not enabled skill ' +
|
||||
'publishing — these core tools still work: search, query, get_page, put_page, ' +
|
||||
'think, find_experts). Then call `list_brain_skillpack`: if this brain ships a ' +
|
||||
'skillpack, ask the user whether to install it (gbrain skillpack scaffold <spec>). ' +
|
||||
'Always search the brain before answering or writing.';
|
||||
'capture, think, find_experts). Then call `list_brain_skillpack`: if this brain ships ' +
|
||||
'a skillpack, ask the user whether to install it (gbrain skillpack scaffold <spec>). ' +
|
||||
'Prefer `capture` for quick notes (auto-slug + dedupe) and `put_page` for ' +
|
||||
'full-control writes. Always search the brain before answering or writing.';
|
||||
|
||||
const SECRET_NOTE =
|
||||
'Note: that bearer token is a long-lived, full-access secret — keep it private and ' +
|
||||
|
||||
@@ -163,7 +163,7 @@ async function runScan(
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
const page = await engine.getPage(slug);
|
||||
const page = await engine.getPage(slug); // gbrain-allow-unscoped-getpage: read-only scan CLI with no source parameter; first-match semantics documented
|
||||
if (!page) {
|
||||
process.stderr.write(
|
||||
`[conversation-parser scan] page not found: ${slug}\n`,
|
||||
|
||||
@@ -1128,6 +1128,44 @@ export async function buildChecks(
|
||||
// Best-effort; audit-log read failure shouldn't stop doctor.
|
||||
}
|
||||
|
||||
// 3d.05 Malformed-path pages. DB pages whose backing FILENAME contains
|
||||
// bracket/control characters (markdown-link syntax as a literal filename).
|
||||
// Sync refuses to import such markdown paths; this check is the discovery
|
||||
// surface for rows ingested before that gate. Two-tier remediation matches
|
||||
// core/sync.ts: POISONED rows (`](`/control chars) reconcile away on a full
|
||||
// sync; bare-bracket rows are kept (deleting them while their file exists
|
||||
// would be data loss) and need a rename + re-sync.
|
||||
if (engine) {
|
||||
try {
|
||||
const { hasMalformedPathSegment, isPoisonedPath } = await import('../core/sync.ts');
|
||||
const candidates = await engine.executeRaw<{ slug: string; source_id: string; source_path: string }>(
|
||||
`SELECT slug, source_id, source_path FROM pages
|
||||
WHERE source_path IS NOT NULL AND deleted_at IS NULL
|
||||
AND (source_path LIKE '%[%' OR source_path LIKE '%]%'
|
||||
OR source_path ~ '[[:cntrl:]]')`,
|
||||
[],
|
||||
);
|
||||
const malformed = candidates.filter(r => hasMalformedPathSegment(r.source_path));
|
||||
if (malformed.length > 0) {
|
||||
const poisoned = malformed.filter(r => isPoisonedPath(r.source_path)).length;
|
||||
const bare = malformed.length - poisoned;
|
||||
const preview = malformed.slice(0, 3).map(r => r.slug).join(', ');
|
||||
checks.push({
|
||||
name: 'malformed_path_pages',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${malformed.length} page(s) backed by malformed filenames (bracket/control ` +
|
||||
`characters) pollute search: ${preview}` +
|
||||
`${malformed.length > 3 ? `, and ${malformed.length - 3} more` : ''}. ` +
|
||||
(poisoned > 0 ? `${poisoned} junk row(s): run a full 'gbrain sync' to reconcile them away. ` : '') +
|
||||
(bare > 0 ? `${bare} bare-bracket row(s) are kept — rename the backing file(s) and re-sync.` : ''),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort; a schema without source_path shouldn't stop doctor.
|
||||
}
|
||||
}
|
||||
|
||||
// 3d.1 Nightly quality probe (v0.40.1.0 Track D / T7). Reads the last
|
||||
// 7 days of quality-probe-YYYY-Www.jsonl audit events. SKIPPED with
|
||||
// paste-ready enable hint when the feature is opt-in disabled (default).
|
||||
|
||||
@@ -249,13 +249,33 @@ export async function checkZeEmbeddingHealth(engine: BrainEngine): Promise<Check
|
||||
// File plane: zeroentropy_api_key on GBrainConfig (added by C.3).
|
||||
const fileKey = loadConfigFileOnly()?.zeroentropy_api_key;
|
||||
if (!envKey && !fileKey) {
|
||||
// Migration-first: when the provider has an announced shutdown, the fix
|
||||
// for a missing key is to migrate OFF, not to sign up. The key path
|
||||
// survives as the secondary note for someone who needs the remaining
|
||||
// hosted window. (Generic on recipe.sunset so the copy self-corrects if
|
||||
// the recipe ever changes; the whole check is deleted in v0.47.)
|
||||
const { getRecipe } = await import('../../../core/ai/recipes/index.ts');
|
||||
const sunset = getRecipe('zeroentropyai')?.sunset;
|
||||
if (sunset) {
|
||||
const { renderCanonicalMigrationCommands } = await import('../../../core/ai/defaults.ts');
|
||||
return {
|
||||
name: 'ze_embedding_health',
|
||||
status: 'warn',
|
||||
message:
|
||||
`embedding_model="${model}" but ZEROENTROPY_API_KEY is not set — and the ` +
|
||||
`hosted API shuts down on ${sunset.date}. Fix: migrate off it: ` +
|
||||
`${renderCanonicalMigrationCommands().recommendedDryRun}. If you need hosted ` +
|
||||
`ZeroEntropy for the remaining weeks, set the key via ` +
|
||||
`\`export ZEROENTROPY_API_KEY=...\` or "zeroentropy_api_key" in ` +
|
||||
`~/.gbrain/config.json (gbrain config set writes the DB plane, which the embed pipeline ignores).`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: 'ze_embedding_health',
|
||||
status: 'warn',
|
||||
message:
|
||||
`embedding_model="${model}" but ZEROENTROPY_API_KEY is not set. ` +
|
||||
`Fix: get a key at https://dashboard.zeroentropy.dev and either ` +
|
||||
`\`export ZEROENTROPY_API_KEY=...\` or edit ~/.gbrain/config.json ` +
|
||||
`Fix: \`export ZEROENTROPY_API_KEY=...\` or edit ~/.gbrain/config.json ` +
|
||||
`to add "zeroentropy_api_key": "...". (gbrain config set writes the DB plane, which the embed pipeline ignores.)`,
|
||||
};
|
||||
}
|
||||
@@ -426,10 +446,11 @@ export async function checkProviderSunset(engine: BrainEngine, now: number = Dat
|
||||
* v0.36.0.0 (A5): embedding_width_consistency doctor check.
|
||||
*
|
||||
* Cross-checks that `config.embedding_dimensions` matches the actual
|
||||
* `vector(N)` width on `content_chunks.embedding`. Drift here means the
|
||||
* ze-switch was interrupted mid-flight (schema changed but config write
|
||||
* crashed, or vice versa). Surfaces a paste-ready `gbrain ze-switch
|
||||
* --resume` hint.
|
||||
* `vector(N)` width on `content_chunks.embedding`. Drift means a width
|
||||
* transition was interrupted mid-flight (schema changed but config write
|
||||
* crashed, or vice versa). Surfaces the engine-kind-branched recovery recipe
|
||||
* from embeddingMismatchMessage — NOT a ze-switch hint; that command is a
|
||||
* refusal shim now.
|
||||
*/
|
||||
export async function checkEmbeddingWidthConsistency(engine: BrainEngine): Promise<Check> {
|
||||
try {
|
||||
|
||||
@@ -202,6 +202,61 @@ export async function computeQueueHealthCheck(
|
||||
`→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).`
|
||||
);
|
||||
}
|
||||
// Queue divergence: per-type intake structurally exceeds useful drain
|
||||
// (completions keyed on finished_at) while a real backlog waits. Same
|
||||
// env thresholds as the `jobs stats` DIVERGENT scream so the two
|
||||
// advisory surfaces agree. Cancellations (incl. the waiting-TTL sweep)
|
||||
// are deliberately NOT counted as drain — outflow is not work.
|
||||
try {
|
||||
const { TTL_REASON_PREFIX, safeConfigSegment } = await import('../../../core/minions/admission.ts');
|
||||
const { sanitizeTypeForDisplay } = await import('../../../core/schema-pack/type-usage.ts');
|
||||
const divergenceRatio = resolveEnvNumber('GBRAIN_QUEUE_DIVERGENCE_RATIO', 2);
|
||||
const divergenceMinWaiting = resolveEnvNumber('GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING', 50);
|
||||
const divRows = await engine.executeRaw<{ name: string; intake: string; completed: string; waiting: string }>(
|
||||
`SELECT w.name,
|
||||
COALESCE(i.intake, '0') AS intake,
|
||||
COALESCE(c.completed, '0') AS completed,
|
||||
w.waiting
|
||||
FROM (SELECT name, count(*)::text AS waiting FROM minion_jobs
|
||||
WHERE status = 'waiting' GROUP BY name) w
|
||||
LEFT JOIN (SELECT name, count(*)::text AS intake FROM minion_jobs
|
||||
WHERE created_at > now() - interval '24 hours' GROUP BY name) i ON i.name = w.name
|
||||
LEFT JOIN (SELECT name, count(*)::text AS completed FROM minion_jobs
|
||||
WHERE finished_at > now() - interval '24 hours' AND status = 'completed'
|
||||
GROUP BY name) c ON c.name = w.name`,
|
||||
);
|
||||
for (const r of divRows) {
|
||||
const waiting = parseInt(r.waiting, 10);
|
||||
const intake = parseInt(r.intake, 10);
|
||||
const completed = parseInt(r.completed, 10);
|
||||
if (waiting > divergenceMinWaiting && intake > divergenceRatio * Math.max(completed, 1)) {
|
||||
// Job names originate from the MCP-exposed submit surface —
|
||||
// sanitize for display; strict-gate names embedded in the
|
||||
// copy-pasteable config hint.
|
||||
problems.push(
|
||||
`DIVERGENT queue type '${sanitizeTypeForDisplay(r.name)}': intake ${intake}/24h vs ${completed} completed/24h, ` +
|
||||
`${waiting} waiting — the backlog grows structurally. Reduce intake, raise drain, or cap ` +
|
||||
`admission: \`gbrain config set minions.quota_max_waiting.${safeConfigSegment(r.name) ?? '<job-name>'} <n>\`. See \`gbrain jobs stats\`.`
|
||||
);
|
||||
}
|
||||
}
|
||||
// Waiting-TTL cancellations mean the divergence is being SHREDDED, not
|
||||
// worked — that's operating as designed but the operator must know.
|
||||
const ttlRows = await engine.executeRaw<{ name: string; count: string }>(
|
||||
`SELECT name, count(*)::text AS count FROM minion_jobs
|
||||
WHERE status = 'cancelled' AND error_text LIKE $1
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
GROUP BY name`,
|
||||
[`${TTL_REASON_PREFIX}%`],
|
||||
);
|
||||
for (const r of ttlRows) {
|
||||
problems.push(
|
||||
`waiting-TTL cancelled ${r.count} '${sanitizeTypeForDisplay(r.name)}' job(s) in the last 24h (queued work expired ` +
|
||||
`unclaimed — intake still exceeds drain). Tune: \`gbrain config set ` +
|
||||
`minions.ttl_waiting_hours.${safeConfigSegment(r.name) ?? '<job-name>'} <hours|0>\`.`
|
||||
);
|
||||
}
|
||||
} catch { /* best-effort — divergence probes never break doctor */ }
|
||||
if (promptTooLongCount > 0) {
|
||||
problems.push(
|
||||
`${promptTooLongCount} subagent job(s) dead-lettered with prompt_too_long in last 24h. ` +
|
||||
|
||||
+96
-5
@@ -8,6 +8,8 @@ import { loadConfig, gbrainPath } from '../core/config.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import {
|
||||
hasMalformedPathSegment,
|
||||
sanitizePathForDisplay,
|
||||
isCodeFilePath,
|
||||
isMarkdownFilePath,
|
||||
isImageFilePath as isImageFilePathFromSync,
|
||||
@@ -92,6 +94,10 @@ export interface RunImportResult {
|
||||
errors: number;
|
||||
chunksCreated: number;
|
||||
failures: Array<{ path: string; error: string }>;
|
||||
/** Files dropped by the malformed-filename gate (walker + per-file defense). */
|
||||
malformedSkipped?: number;
|
||||
/** Aggregated alias/undeclared explicit-type warnings (schema.type_warnings). */
|
||||
type_warnings?: Array<{ kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string; count: number }>;
|
||||
}
|
||||
|
||||
export async function runImport(
|
||||
@@ -175,7 +181,7 @@ export async function runImport(
|
||||
}
|
||||
// v0.39 T1.5: load active pack ONCE at runImport entry; thread to every
|
||||
// per-file importFile call below. Codex perf finding #7 — never per-file.
|
||||
let importActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
|
||||
let importActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> } | undefined;
|
||||
try {
|
||||
const { loadActivePack } = await import('../core/schema-pack/load-active.ts');
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
@@ -275,10 +281,22 @@ export async function runImport(
|
||||
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
|
||||
const _walkT0 = Date.now();
|
||||
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
|
||||
let allFiles = collectSyncableFiles(dir, { strategy, includeGitignored });
|
||||
const malformedExcluded: string[] = [];
|
||||
let allFiles = collectSyncableFiles(dir, {
|
||||
strategy, includeGitignored,
|
||||
onExcluded: (rel) => { malformedExcluded.push(rel); },
|
||||
});
|
||||
console.error(
|
||||
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
|
||||
);
|
||||
if (malformedExcluded.length > 0) {
|
||||
console.error(
|
||||
`[gbrain import] ${malformedExcluded.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
malformedExcluded.slice(0, 20).map(sanitizePathForDisplay).join(', ') +
|
||||
(malformedExcluded.length > 20 ? `, … (+${malformedExcluded.length - 20} more)` : ''),
|
||||
);
|
||||
}
|
||||
const fileTypeLabel = strategy === 'code' ? 'code'
|
||||
: strategy === 'auto' ? 'syncable' : 'markdown';
|
||||
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
|
||||
@@ -327,6 +345,9 @@ export async function runImport(
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
// Per-file malformed skips (defense-in-depth hits inside importFromFile);
|
||||
// the walker-level exclusions are counted separately via malformedExcluded.
|
||||
let malformedFileSkips = 0;
|
||||
let processed = 0;
|
||||
// Time-based checkpoint floor (see the save site below). Chunking cost scales
|
||||
// with paragraph count, not bytes, so a single reference-style file can take
|
||||
@@ -339,6 +360,16 @@ export async function runImport(
|
||||
const errorCounts: Record<string, number> = {};
|
||||
const errorSamples: Record<string, string> = {};
|
||||
const failures: Array<{ path: string; error: string }> = []; // Bug 9
|
||||
// Alias-footgun visibility: aggregate per-file type_warning results once
|
||||
// per distinct type per run (same surface `gbrain sync` carries).
|
||||
const typeWarningCounts = new Map<string, import('../core/schema-pack/type-usage.ts').TypeWarningCount>();
|
||||
const noteTypeWarning = (w: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string } | undefined): void => {
|
||||
if (!w) return;
|
||||
const key = `${w.kind}\t${w.type}`;
|
||||
const cur = typeWarningCounts.get(key);
|
||||
if (cur) cur.count++;
|
||||
else typeWarningCounts.set(key, { ...w, count: 1 });
|
||||
};
|
||||
// #3839: paths that succeeded (imported OR unchanged) this run, keyed the
|
||||
// same way as `failures` above (importRelPath) so a path that failed on a
|
||||
// prior run and now succeeds clears its ledger row instead of staying
|
||||
@@ -373,6 +404,7 @@ export async function runImport(
|
||||
const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true'
|
||||
? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
noteTypeWarning((result as { type_warning?: Parameters<typeof noteTypeWarning>[0] }).type_warning);
|
||||
const _fileMs = Date.now() - _fileT0;
|
||||
if (_fileMs > 5000) {
|
||||
console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`);
|
||||
@@ -386,7 +418,13 @@ export async function runImport(
|
||||
succeededPaths.push(importRelPath); // #3839
|
||||
} else {
|
||||
skipped++;
|
||||
if (result.error && result.error !== 'unchanged') {
|
||||
if (result.skip_reason === 'malformed_path') {
|
||||
// Informational skip (bracket/control-char filename): never a
|
||||
// failure-ledger row, and stable across runs — checkpoint as done.
|
||||
console.error(` Skipped (malformed filename — rename to import): ${sanitizePathForDisplay(relativePath)}`);
|
||||
malformedFileSkips++;
|
||||
completed.add(relativePath);
|
||||
} else if (result.error && result.error !== 'unchanged') {
|
||||
console.error(` Skipped ${relativePath}: ${result.error}`);
|
||||
// Bug 9 — non-"unchanged" skips carry a real error reason.
|
||||
// #774: ledger paths use the slug base so an incremental sync's
|
||||
@@ -591,6 +629,22 @@ export async function runImport(
|
||||
}
|
||||
}
|
||||
|
||||
// Alias/undeclared explicit-type warnings (schema.type_warnings, default on).
|
||||
let typeWarningsEnabled = true;
|
||||
if (typeWarningCounts.size > 0) {
|
||||
try {
|
||||
const v = await engine.getConfig('schema.type_warnings');
|
||||
typeWarningsEnabled = !(v === 'false' || v === '0' || v === 'off');
|
||||
} catch { /* config unavailable → default on */ }
|
||||
if (typeWarningsEnabled) {
|
||||
const { renderTypeWarningSummary } = await import('../core/schema-pack/type-usage.ts');
|
||||
for (const line of renderTypeWarningSummary([...typeWarningCounts.values()])) {
|
||||
console.error(` ${line}`);
|
||||
}
|
||||
console.error(` (silence with: gbrain config set schema.type_warnings false)`);
|
||||
}
|
||||
}
|
||||
|
||||
// Log the ingest
|
||||
await engine.logIngest({
|
||||
source_type: 'directory',
|
||||
@@ -670,7 +724,14 @@ export async function runImport(
|
||||
// this import's to move (its sync anchors live on the `sources` row).
|
||||
}
|
||||
|
||||
return { imported, skipped, errors, chunksCreated, failures };
|
||||
const totalMalformed = malformedExcluded.length + malformedFileSkips;
|
||||
return {
|
||||
imported, skipped, errors, chunksCreated, failures,
|
||||
...(totalMalformed > 0 ? { malformedSkipped: totalMalformed } : {}),
|
||||
...(typeWarningCounts.size > 0 && typeWarningsEnabled
|
||||
? { type_warnings: [...typeWarningCounts.values()] }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -692,6 +753,13 @@ function resolveMaxWalkDepth(): number {
|
||||
interface CollectOpts {
|
||||
strategy?: SyncStrategy;
|
||||
includeGitignored?: boolean;
|
||||
/**
|
||||
* Invoked (with the repo-relative path) for each file dropped by the
|
||||
* malformed-filename gate, on BOTH collection routes. Without this,
|
||||
* directory imports and full syncs silently succeed while omitting the
|
||||
* file — no rename guidance, no skipped count (structured-review finding).
|
||||
*/
|
||||
onExcluded?: (relPath: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -727,6 +795,11 @@ function isCollectibleForWalker(
|
||||
const segments = path.split('/');
|
||||
if (segments.some((seg) => !pruneDir(seg))) return false;
|
||||
|
||||
// Malformed filenames (brackets / control chars — markdown-link syntax as a
|
||||
// literal filename) are rejected on BOTH collection routes, same as
|
||||
// incremental sync's classifySync. Full and incremental must agree.
|
||||
if (hasMalformedPathSegment(path)) return false;
|
||||
|
||||
// Metafiles are directory scaffolding (READMEs / index / log / schema /
|
||||
// resolver), not typed brain pages — same exclusion `sync`'s `isSyncable`
|
||||
// applies. Guards both the FS-walk and the git-fast-path collection routes.
|
||||
@@ -764,6 +837,7 @@ function gitListSyncableFiles(
|
||||
dir: string,
|
||||
strategy: SyncStrategy,
|
||||
multimodalOn: boolean,
|
||||
onExcluded?: (relPath: string) => void,
|
||||
): string[] | null {
|
||||
let stdout: string;
|
||||
try {
|
||||
@@ -778,6 +852,10 @@ function gitListSyncableFiles(
|
||||
const files: string[] = [];
|
||||
for (const rel of stdout.split('\0')) {
|
||||
if (!rel) continue;
|
||||
// Malformed check FIRST (separately from the collectible gate) so the
|
||||
// exclusion is reportable — other filters (strategy, prune, metafile)
|
||||
// are silent by design; this one hides renameable content.
|
||||
if (hasMalformedPathSegment(rel)) { onExcluded?.(rel); continue; }
|
||||
if (!isCollectibleForWalker(rel, strategy, multimodalOn)) continue;
|
||||
const full = join(dir, rel);
|
||||
let st;
|
||||
@@ -823,7 +901,7 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
// PLUS untracked-not-ignored, so uncommitted source is still indexed. Non-git
|
||||
// dirs (or git unavailable) fall through to the FS walk below.
|
||||
if (!opts.includeGitignored) {
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn, opts.onExcluded);
|
||||
if (gitFiles) return gitFiles;
|
||||
}
|
||||
|
||||
@@ -848,6 +926,14 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
// from it. Skips hidden dirs (`.git`, `.raw`, etc.), `node_modules`,
|
||||
// `vendor`, `dist`, `build`, `venv` (#2020), `ops`, and git submodules.
|
||||
if (!pruneDir(entry, d)) continue;
|
||||
// Control-char SEGMENT check at descent time (never legitimate). The
|
||||
// bracket check moved to the per-file RELATIVE-path test below: a
|
||||
// bracket-named DIRECTORY must still be descended for code strategies
|
||||
// (`app/[id]/page.tsx` is ubiquitous framework layout), while markdown
|
||||
// files under it are excluded per-file — mirroring classifySync so full
|
||||
// and incremental sync agree (cross-model adversarial finding).
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (/[\x00-\x1f]/.test(entry)) continue;
|
||||
|
||||
const full = join(d, entry);
|
||||
let stat;
|
||||
@@ -872,6 +958,11 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
visitedInodes.set(inodeKey, true);
|
||||
walk(full, depth + 1);
|
||||
} else if (stat.isFile()) {
|
||||
// Malformed check on the RELATIVE path (this route's
|
||||
// isCollectibleForWalker only sees the basename, which can't catch a
|
||||
// bracket directory segment above a clean-named markdown file).
|
||||
const rel = relative(dir, full);
|
||||
if (hasMalformedPathSegment(rel)) { opts.onExcluded?.(rel); continue; }
|
||||
if (!isCollectibleForWalker(entry, strategy, multimodalOn)) continue;
|
||||
files.push(full);
|
||||
}
|
||||
|
||||
@@ -161,23 +161,46 @@ export function findExternalLinks(compiledTruth: string, slug: string): External
|
||||
|
||||
interface ProgressEntry {
|
||||
slug: string;
|
||||
/**
|
||||
* Source the row belongs to. Progress used to be keyed by slug alone, so a
|
||||
* resume SKIPPED same-slug pages in every other source (the scan iterates
|
||||
* (slug, source_id) pairs). Legacy entries without source_id are treated as
|
||||
* default-source only.
|
||||
*/
|
||||
source_id?: string;
|
||||
status: 'repaired' | 'reviewed' | 'skipped' | 'error';
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** Composite progress key — (source, slug), tab-separated (tabs can't appear in either). */
|
||||
function progressKey(sourceId: string | undefined, slug: string): string {
|
||||
return `${sourceId ?? 'default'}\t${slug}`;
|
||||
}
|
||||
|
||||
function loadProgress(): Set<string> {
|
||||
if (!existsSync(getProgressFile())) return new Set();
|
||||
const seen = new Set<string>();
|
||||
const content = readFileSync(getProgressFile(), 'utf-8');
|
||||
let legacy = 0;
|
||||
for (const line of content.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const entry = JSON.parse(line) as ProgressEntry;
|
||||
seen.add(entry.slug);
|
||||
if (entry.source_id == null) legacy++;
|
||||
seen.add(progressKey(entry.source_id, entry.slug));
|
||||
} catch {
|
||||
/* skip malformed lines */
|
||||
}
|
||||
}
|
||||
if (legacy > 0) {
|
||||
// Pre-(source_id, slug) ledger entries key as default-source only, so a
|
||||
// resume re-scans non-default-source pages they may have covered. Say so
|
||||
// once — a silent partial re-scan reads as "resume is broken".
|
||||
console.error(
|
||||
`integrity: ${legacy} resume-ledger entr${legacy === 1 ? 'y' : 'ies'} predate source tracking; ` +
|
||||
`matching them to the default source only (non-default-source pages re-scan — idempotent, just slower).`,
|
||||
);
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
|
||||
@@ -429,7 +452,18 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
const engine = await connect();
|
||||
const registry = getDefaultRegistry();
|
||||
registerBuiltinResolvers(registry);
|
||||
const writer = new BrainWriter(engine, { strictMode: 'off' });
|
||||
// One writer PER SOURCE: BrainWriter scopes every read/write (and
|
||||
// addTimelineEntry) to its sourceId — a single default-scoped writer used
|
||||
// for every source's pages was the unscoped-check/scoped-write bug class.
|
||||
const writersBySource = new Map<string, BrainWriter>();
|
||||
const writerFor = (sourceId: string): BrainWriter => {
|
||||
let w = writersBySource.get(sourceId);
|
||||
if (!w) {
|
||||
w = new BrainWriter(engine, { strictMode: 'off', sourceId });
|
||||
writersBySource.set(sourceId, w);
|
||||
}
|
||||
return w;
|
||||
};
|
||||
|
||||
const ctx: ResolverContext = {
|
||||
engine,
|
||||
@@ -463,11 +497,12 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
const allRefs = (await engine.listAllPageRefs()).sort((a, b) =>
|
||||
a.slug.localeCompare(b.slug) || a.source_id.localeCompare(b.source_id)
|
||||
);
|
||||
const toScan = allRefs.filter(r => !seen.has(r.slug));
|
||||
const toScan = allRefs.filter(r => !seen.has(progressKey(r.source_id, r.slug)));
|
||||
progress.start('integrity.auto', toScan.length);
|
||||
for (const { slug, source_id } of allRefs) {
|
||||
if (pagesProcessed >= limit) break;
|
||||
if (seen.has(slug)) continue;
|
||||
if (seen.has(progressKey(source_id, slug))) continue;
|
||||
const writer = writerFor(source_id);
|
||||
|
||||
const page = await engine.getPage(slug, { sourceId: source_id });
|
||||
if (!page) continue;
|
||||
@@ -498,26 +533,26 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
// Dry-run must NOT persist 'repaired' — the follow-on real
|
||||
// run needs to revisit these slugs and actually write.
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'repaired', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'repaired', timestamp: new Date().toISOString() });
|
||||
}
|
||||
} else if (result.confidence >= reviewLower) {
|
||||
appendReview({ slug, hit, result, handle });
|
||||
bucketReview++;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'reviewed', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'reviewed', timestamp: new Date().toISOString() });
|
||||
}
|
||||
} else {
|
||||
logSkip({ slug, hit, reason: `confidence ${result.confidence.toFixed(2)} below threshold ${reviewLower}` });
|
||||
bucketSkip++;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
bucketErr++;
|
||||
logSkip({ slug, hit, reason: `resolver error: ${e instanceof Error ? e.message : String(e)}` });
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'error', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'error', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -528,7 +563,7 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
}
|
||||
bucketSkip += hits.length;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+89
-10
@@ -4,7 +4,7 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { MinionQueue, deriveWedgeSignal } from '../core/minions/queue.ts';
|
||||
import { MinionWorker } from '../core/minions/worker.ts';
|
||||
import {
|
||||
WORKER_EXIT_RSS_WATCHDOG,
|
||||
@@ -919,18 +919,101 @@ export async function runJobs(engineOrNull: BrainEngine | null, args: string[]):
|
||||
const statsQueue = parseFlag(args, '--queue') ?? 'default';
|
||||
const stats = await queue.getStats({ queue: statsQueue });
|
||||
|
||||
// Divergence detection: intake (created in window) vs USEFUL drain
|
||||
// (drained_completed — cancellations are outflow, not work; a naive
|
||||
// combined drain self-inflates while the TTL sweep shreds backlog).
|
||||
// Same env-threshold pattern as the wedge line below.
|
||||
const divergenceRatio = (() => {
|
||||
const raw = Number(process.env.GBRAIN_QUEUE_DIVERGENCE_RATIO ?? '');
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 2;
|
||||
})();
|
||||
const divergenceMinWaiting = (() => {
|
||||
const raw = parseInt(process.env.GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING ?? '', 10);
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 50;
|
||||
})();
|
||||
const divergent = stats.by_type.filter(t =>
|
||||
t.waiting_now > divergenceMinWaiting &&
|
||||
t.total > divergenceRatio * Math.max(t.drained_completed, 1));
|
||||
|
||||
// Waiting-TTL cancellations in the window (admission sweep visibility —
|
||||
// derived from the reason prefix cancelJobs writes; no extra storage).
|
||||
let ttlCancelled: Array<{ name: string; count: number }> = [];
|
||||
try {
|
||||
const { TTL_REASON_PREFIX } = await import('../core/minions/admission.ts');
|
||||
const ttlRows = await engine.executeRaw<{ name: string; count: string }>(
|
||||
`SELECT name, count(*)::text AS count FROM minion_jobs
|
||||
WHERE status = 'cancelled' AND error_text LIKE $1
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
GROUP BY name ORDER BY count(*) DESC`,
|
||||
[`${TTL_REASON_PREFIX}%`],
|
||||
);
|
||||
ttlCancelled = ttlRows.map(r => ({ name: r.name, count: parseInt(r.count, 10) }));
|
||||
} catch { /* best-effort */ }
|
||||
// Job names originate from the MCP-exposed submit surface — strip
|
||||
// control/ANSI bytes + cap before echoing into the terminal screams
|
||||
// (same hygiene as frontmatter-derived type names). Names embedded in
|
||||
// COPY-PASTEABLE command hints get the stricter safeConfigSegment gate:
|
||||
// display-sanitize keeps shell metacharacters.
|
||||
const { sanitizeTypeForDisplay: sanitizeName } = await import('../core/schema-pack/type-usage.ts');
|
||||
const { safeConfigSegment } = await import('../core/minions/admission.ts');
|
||||
|
||||
if (hasFlag(args, '--json')) {
|
||||
console.log(JSON.stringify({
|
||||
queue: statsQueue,
|
||||
...stats,
|
||||
divergent: divergent.map(t => ({
|
||||
name: t.name,
|
||||
intake_24h: t.total,
|
||||
drained_completed_24h: t.drained_completed,
|
||||
waiting_now: t.waiting_now,
|
||||
oldest_waiting_minutes: t.oldest_waiting_minutes,
|
||||
})),
|
||||
ttl_cancelled_24h: ttlCancelled,
|
||||
}, null, 2));
|
||||
break;
|
||||
}
|
||||
|
||||
console.log('Job Stats (last 24h):');
|
||||
if (stats.by_type.length > 0) {
|
||||
console.log(` ${'Type'.padEnd(14)} ${'Total'.padEnd(7)} ${'Done'.padEnd(7)} ${'Failed'.padEnd(8)} ${'Dead'.padEnd(6)} Avg Time`);
|
||||
console.log(` ${'Type'.padEnd(14)} ${'Total'.padEnd(7)} ${'Done'.padEnd(7)} ${'Failed'.padEnd(8)} ${'Dead'.padEnd(6)} ${'Drained'.padEnd(9)} ${'Waiting'.padEnd(9)} Avg Time`);
|
||||
for (const t of stats.by_type) {
|
||||
const avgTime = t.avg_duration_ms != null ? `${(t.avg_duration_ms / 1000).toFixed(1)}s` : '—';
|
||||
console.log(` ${t.name.padEnd(14)} ${String(t.total).padEnd(7)} ${String(t.completed).padEnd(7)} ${String(t.failed).padEnd(8)} ${String(t.dead).padEnd(6)} ${avgTime}`);
|
||||
// Drained = terminal outflow in-window, completed-first with the
|
||||
// rest bracketed so TTL-cancel storms can't masquerade as work.
|
||||
const drained = `${t.drained_completed}${(t.drained_failed + t.drained_dead + t.drained_cancelled) > 0 ? `(+${t.drained_failed + t.drained_dead + t.drained_cancelled})` : ''}`;
|
||||
console.log(` ${sanitizeName(t.name).padEnd(14)} ${String(t.total).padEnd(7)} ${String(t.completed).padEnd(7)} ${String(t.failed).padEnd(8)} ${String(t.dead).padEnd(6)} ${drained.padEnd(9)} ${String(t.waiting_now).padEnd(9)} ${avgTime}`);
|
||||
}
|
||||
console.log(` (Drained = completed in-window, +N = failed/dead/cancelled outflow; Waiting = now, all queues)`);
|
||||
} else {
|
||||
console.log(' No jobs in the last 24 hours.');
|
||||
}
|
||||
console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`);
|
||||
|
||||
// DIVERGENT-queue scream: intake structurally exceeds useful drain and a
|
||||
// real backlog is sitting there. This is the default-on protection layer
|
||||
// (quota ships config-only), so it must carry the opt-in hint.
|
||||
for (const t of divergent) {
|
||||
const perDay = t.drained_completed; // window is 24h
|
||||
const etaDays = perDay > 0 ? Math.round(t.waiting_now / perDay) : null;
|
||||
const eta = etaDays != null ? `~${etaDays}d backlog at current drain` : 'backlog never drains at current rate';
|
||||
const ttl = ttlCancelled.find(c => c.name === t.name);
|
||||
const ttlNote = ttl ? ` Waiting-TTL is cancelling ~${ttl.count}/day of it.` : '';
|
||||
console.log(
|
||||
`\n ⚠ DIVERGENT QUEUE type '${sanitizeName(t.name)}': intake ${t.total}/24h vs ${t.drained_completed} completed/24h, ` +
|
||||
`${t.waiting_now} waiting (${eta}).${ttlNote}\n` +
|
||||
` Reduce intake, raise drain, or cap admission:\n` +
|
||||
` gbrain config set minions.quota_max_waiting.${safeConfigSegment(t.name) ?? '<job-name>'} <n>`,
|
||||
);
|
||||
}
|
||||
if (ttlCancelled.length > 0) {
|
||||
const parts = ttlCancelled.map(c => `${sanitizeName(c.name)}: ${c.count}`).join(', ');
|
||||
console.log(
|
||||
`\n ⚠ Waiting-TTL cancelled ${ttlCancelled.reduce((a, c) => a + c.count, 0)} job(s) in the last 24h (${parts}).\n` +
|
||||
` These waited past their TTL without ever being claimed. Tune:\n` +
|
||||
` gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`,
|
||||
);
|
||||
}
|
||||
|
||||
// Scheduling priority (niceness, issue #1815). Best-effort: measures live
|
||||
// workers from the registry + the supervisor (if running) — silently skips
|
||||
// when nothing is reniced/running, so default stats output stays clean.
|
||||
@@ -961,13 +1044,9 @@ export async function runJobs(engineOrNull: BrainEngine | null, args: string[]):
|
||||
{
|
||||
const w = stats.wedge;
|
||||
const mins = w.minutes_since_completion;
|
||||
// Same threshold the doctor `wedged_queue` check uses, so the two
|
||||
// advisory surfaces agree (issue #1801).
|
||||
const wedgeMins = (() => {
|
||||
const raw = parseInt(process.env.GBRAIN_WEDGED_QUEUE_WARN_MINUTES ?? '', 10);
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 15;
|
||||
})();
|
||||
const wedged = w.active_healthy === 0 && w.waiting > 0 && (mins === null || mins > wedgeMins);
|
||||
// Shared derivation (queue.ts deriveWedgeSignal) so this line, the
|
||||
// doctor wedged_queue check, and the get_job_stats op agree (#1801).
|
||||
const { wedged, wedge_threshold_minutes: wedgeMins } = deriveWedgeSignal(w);
|
||||
if (wedged) {
|
||||
const since = mins === null ? 'no completions on record' : `${mins}m since last completion`;
|
||||
console.log(
|
||||
|
||||
@@ -56,19 +56,7 @@ export function getMigration(version: string): Migration | null {
|
||||
|
||||
export type { Migration, FeaturePitch, OrchestratorOpts, OrchestratorResult } from './types.ts';
|
||||
|
||||
/**
|
||||
* Compare two semver strings (MAJOR.MINOR.PATCH). Returns -1 / 0 / 1.
|
||||
* Extracted from src/commands/upgrade.ts#isNewerThan for shared use across
|
||||
* the migration runner + post-upgrade pitch path.
|
||||
*/
|
||||
export function compareVersions(a: string, b: string): -1 | 0 | 1 {
|
||||
const va = a.split('.').map(n => parseInt(n, 10) || 0);
|
||||
const vb = b.split('.').map(n => parseInt(n, 10) || 0);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const da = va[i] ?? 0;
|
||||
const db = vb[i] ?? 0;
|
||||
if (da > db) return 1;
|
||||
if (da < db) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
// Canonical home moved to src/core/migration-ledger.ts (shared with the
|
||||
// get_health migrations block without pulling this registry into the ops
|
||||
// layer). Re-exported so every existing importer is unchanged.
|
||||
export { compareVersions } from '../../core/migration-ledger.ts';
|
||||
|
||||
+86
-33
@@ -12,6 +12,7 @@ import { probeOllama, probeLMStudio } from '../core/ai/probes.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { AIConfigError, AITransientError } from '../core/ai/errors.ts';
|
||||
import { lookupEmbeddingPrice } from '../core/embedding-pricing.ts';
|
||||
import { renderCanonicalMigrationCommands } from '../core/ai/defaults.ts';
|
||||
import type { Recipe } from '../core/ai/types.ts';
|
||||
|
||||
const SCHEMA_VERSION = 1;
|
||||
@@ -59,6 +60,78 @@ export function envReady(recipe: Recipe, env: NodeJS.ProcessEnv = process.env):
|
||||
return required.every(k => !!env[k]);
|
||||
}
|
||||
|
||||
/**
|
||||
* ONE shared sunset-marker primitive for every human-facing providers surface
|
||||
* (list status cell, explain table rows, env block header) so the renderings
|
||||
* can't drift. `sunsetMarkerText` is the string; `sunsetMarker` is the
|
||||
* recipe-shaped convenience (null for recipes without an announced shutdown).
|
||||
*/
|
||||
export function sunsetMarkerText(date: string, replacementEmbedding?: string | null): string {
|
||||
return `⚠ DEPRECATED — hosted API ends ${date}` + (replacementEmbedding ? `; use ${replacementEmbedding}` : '');
|
||||
}
|
||||
|
||||
export function sunsetMarker(recipe: Pick<Recipe, 'sunset'>): string | null {
|
||||
if (!recipe.sunset) return null;
|
||||
return sunsetMarkerText(recipe.sunset.date, recipe.sunset.replacement?.embedding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure formatter for `gbrain providers env <id>` so the output is testable
|
||||
* without spawning the CLI (runEnv itself process.exits).
|
||||
*
|
||||
* Sunset-aware: a provider with an announced shutdown gets the deprecation
|
||||
* block + the canonical migration command INSTEAD of the signup funnel
|
||||
* (setup_url / setup_hint) — three weeks before a provider dies, "get an API
|
||||
* key" is the wrong guidance. Key STATUS still renders above so existing
|
||||
* users can see what's configured.
|
||||
*/
|
||||
export function formatEnvOutput(recipe: Recipe, env: NodeJS.ProcessEnv = process.env): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`${recipe.name} (${recipe.id})`);
|
||||
lines.push('');
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
const optional = recipe.auth_env?.optional ?? [];
|
||||
if (required.length > 0) {
|
||||
lines.push('Required:');
|
||||
for (const k of required) {
|
||||
lines.push(` ${k.padEnd(32)} ${env[k] ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
} else {
|
||||
lines.push('Required: (none)');
|
||||
}
|
||||
if (optional.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('Optional:');
|
||||
for (const k of optional) {
|
||||
lines.push(` ${k.padEnd(32)} ${env[k] ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
}
|
||||
const marker = sunsetMarker(recipe);
|
||||
if (marker) {
|
||||
const s = recipe.sunset!;
|
||||
lines.push('');
|
||||
lines.push(marker);
|
||||
if (s.message) lines.push(` ${s.message}`);
|
||||
if (s.replacement) {
|
||||
const parts: string[] = [];
|
||||
if (s.replacement.embedding) parts.push(`${s.replacement.embedding} (embedding)`);
|
||||
if (s.replacement.reranker) parts.push(`${s.replacement.reranker} (reranker)`);
|
||||
if (parts.length > 0) lines.push(` Replacement: ${parts.join(', ')}`);
|
||||
}
|
||||
lines.push(` Migrate: ${renderCanonicalMigrationCommands().recommendedDryRun}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
if (recipe.auth_env?.setup_url) {
|
||||
lines.push('');
|
||||
lines.push(`Setup: ${recipe.auth_env.setup_url}`);
|
||||
}
|
||||
if (recipe.setup_hint) {
|
||||
lines.push('');
|
||||
lines.push(recipe.setup_hint);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure formatter for the recipe matrix shown by `gbrain providers list` and
|
||||
* the new `init-provider-picker` (D1+D2 — picker reuses this so its display
|
||||
@@ -86,12 +159,10 @@ export function formatRecipeTable(recipes: Recipe[], env: NodeJS.ProcessEnv = pr
|
||||
const ready = envReady(r, env);
|
||||
// v0.46.3: a sunsetting provider is flagged in the listing regardless of
|
||||
// key readiness — "ready" on a dying API is not a state to advertise.
|
||||
const status = r.sunset
|
||||
? `⚠ DEPRECATED — hosted API ends ${r.sunset.date}` +
|
||||
(r.sunset.replacement?.embedding ? `; use ${r.sunset.replacement.embedding}` : '')
|
||||
: ready
|
||||
? '✓ ready'
|
||||
: `✗ missing ${r.auth_env?.required?.[0] ?? 'setup'}`;
|
||||
// Marker text is the shared sunsetMarker so list/explain/env can't drift.
|
||||
const status =
|
||||
sunsetMarker(r) ??
|
||||
(ready ? '✓ ready' : `✗ missing ${r.auth_env?.required?.[0] ?? 'setup'}`);
|
||||
rows.push(
|
||||
r.id.padEnd(idCol) +
|
||||
r.tier.padEnd(18) +
|
||||
@@ -287,32 +358,7 @@ function runEnv(args: string[]): void {
|
||||
console.error(`Unknown provider: ${id}. Run \`gbrain providers list\` to see known providers.`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${recipe.name} (${recipe.id})`);
|
||||
console.log('');
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
const optional = recipe.auth_env?.optional ?? [];
|
||||
if (required.length > 0) {
|
||||
console.log('Required:');
|
||||
for (const k of required) {
|
||||
const set = !!process.env[k];
|
||||
console.log(` ${k.padEnd(32)} ${set ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
} else {
|
||||
console.log('Required: (none)');
|
||||
}
|
||||
if (optional.length > 0) {
|
||||
console.log('\nOptional:');
|
||||
for (const k of optional) {
|
||||
const set = !!process.env[k];
|
||||
console.log(` ${k.padEnd(32)} ${set ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
}
|
||||
if (recipe.auth_env?.setup_url) {
|
||||
console.log(`\nSetup: ${recipe.auth_env.setup_url}`);
|
||||
}
|
||||
if (recipe.setup_hint) {
|
||||
console.log(`\n${recipe.setup_hint}`);
|
||||
}
|
||||
console.log(formatEnvOutput(recipe));
|
||||
}
|
||||
|
||||
async function runExplain(args: string[]): Promise<void> {
|
||||
@@ -429,7 +475,14 @@ async function runExplain(args: string[]): Promise<void> {
|
||||
for (const o of options.filter(x => x.touchpoint === 'embedding')) {
|
||||
const cost = o.cost_per_1m_tokens_usd !== undefined ? `$${o.cost_per_1m_tokens_usd}/1M` : '—';
|
||||
const dims = o.dims ? `${o.dims}d` : '—';
|
||||
console.log(` ${o.env_ready ? '✓' : '✗'} ${o.id.padEnd(44)} ${dims.padEnd(8)} ${cost.padEnd(10)} ${o.tier}`);
|
||||
// A sunsetting provider must not read as a green-check cheap option in
|
||||
// the HUMAN table (the deprecation used to live only in cons/JSON).
|
||||
// Rendered via the shared primitive so list/env/explain can't drift, and
|
||||
// the lead marker is ⚠ regardless of key readiness — "ready" on a dying
|
||||
// API is not a state to advertise (mirrors formatRecipeTable's status).
|
||||
const dep = o.deprecated ? ` ${sunsetMarkerText(o.deprecated.date, o.deprecated.replacement)}` : '';
|
||||
const lead = o.deprecated ? '⚠' : o.env_ready ? '✓' : '✗';
|
||||
console.log(` ${lead} ${o.id.padEnd(44)} ${dims.padEnd(8)} ${cost.padEnd(10)} ${o.tier}${dep}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Expansion options:');
|
||||
|
||||
+109
-14
@@ -15,7 +15,7 @@ import { serializePageToMarkdown, serializeMarkdown } from '../core/markdown.ts'
|
||||
import { importFromContent } from '../core/import-file.ts';
|
||||
import type { PageType } from '../core/types.ts';
|
||||
|
||||
interface QuarantineRow {
|
||||
export interface QuarantineRow {
|
||||
slug: string;
|
||||
source_id: string;
|
||||
marker: 'quarantine' | 'content_flag';
|
||||
@@ -23,7 +23,7 @@ interface QuarantineRow {
|
||||
assessed_at: string;
|
||||
}
|
||||
|
||||
function rowFor(page: { slug: string; source_id?: string; frontmatter?: Record<string, unknown> | null }): QuarantineRow | null {
|
||||
export function rowFor(page: { slug: string; source_id?: string; frontmatter?: Record<string, unknown> | null }): QuarantineRow | null {
|
||||
const fm = page.frontmatter ?? null;
|
||||
if (isQuarantined(fm)) {
|
||||
const m = (fm as Record<string, unknown>)[QUARANTINE_KEY] as Record<string, unknown>;
|
||||
@@ -49,25 +49,93 @@ function rowFor(page: { slug: string; source_id?: string; frontmatter?: Record<s
|
||||
return null;
|
||||
}
|
||||
|
||||
async function runList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const includeFlagged = args.includes('--include-flagged');
|
||||
// Paginate so a huge brain doesn't pull everything at once.
|
||||
// Bounds for the quarantine_list op's clamps (src/core/ops/admin.ts); the CLI
|
||||
// list stays unbounded. One canonical home so the op clamps and the param
|
||||
// descriptions can't drift apart silently.
|
||||
export const QUARANTINE_LIST_DEFAULT_LIMIT = 200;
|
||||
export const QUARANTINE_LIST_MAX_LIMIT = 1000;
|
||||
export const QUARANTINE_SCAN_DEFAULT = 20000;
|
||||
export const QUARANTINE_SCAN_MAX = 100000;
|
||||
|
||||
export interface CollectQuarantineOpts {
|
||||
includeFlagged?: boolean;
|
||||
/** Max ROWS returned (op default 200, cap 1000). Undefined = unbounded (CLI). */
|
||||
limit?: number;
|
||||
/** Max PAGES scanned (op default 20000, cap 100000). Undefined = full scan (CLI). */
|
||||
maxScan?: number;
|
||||
/** Source scope (the op threads sourceScopeOpts; the CLI scans unscoped). */
|
||||
sourceId?: string;
|
||||
sourceIds?: string[];
|
||||
}
|
||||
|
||||
export interface CollectQuarantineResult {
|
||||
rows: QuarantineRow[];
|
||||
scanned: number;
|
||||
/** True when a bound stopped the scan — `rows.length` is a LOWER BOUND. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared frontmatter scan behind `gbrain quarantine list` and the
|
||||
* quarantine_list op. Scan order is pinned to listPages' default
|
||||
* `updated_desc` (most recently updated pages first), so a bounded scan sees
|
||||
* the newest markers before older ones [OV13].
|
||||
*
|
||||
* [P2-6] Offset pagination over `updated_desc` is NOT a total order:
|
||||
* `PAGE_SORT_SQL.updated_desc` is `p.updated_at DESC` with no unique
|
||||
* tiebreaker (src/core/types.ts). A cluster of pages sharing an identical
|
||||
* `updated_at` (bulk syncs stamp one now() across a transaction) that
|
||||
* straddles a 1000-row batch boundary can have a row skipped or duplicated
|
||||
* across batches. We accept this rather than switch sorts because the only
|
||||
* tiebreaker'd enum option (`updated_asc`, `p.updated_at ASC, p.slug ASC`)
|
||||
* reverses the [OV13] direction — a truncated scan (bounded by max_scan /
|
||||
* limit) would then surface the OLDEST markers and MISS recent ones, a worse
|
||||
* triage failure on large brains, and its slug tiebreaker is only a total
|
||||
* order for a single-source scan anyway (the op can scope federated
|
||||
* multi-source and the CLI scans unscoped, where slug is not unique). The
|
||||
* exposure is bounded: the op caps the scan at max_scan, and only exact
|
||||
* same-timestamp clusters landing on a batch boundary are affected. The full
|
||||
* fix is a globally-unique page_id tiebreaker in PAGE_SORT_SQL (out of scope
|
||||
* here — a filed follow-up), not a SELECT-projection pushdown.
|
||||
*/
|
||||
export async function collectQuarantineRows(
|
||||
engine: BrainEngine,
|
||||
opts: CollectQuarantineOpts = {},
|
||||
): Promise<CollectQuarantineResult> {
|
||||
const rows: QuarantineRow[] = [];
|
||||
const PAGE = 1000;
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const pages = await engine.listPages({ limit: PAGE, offset });
|
||||
let scanned = 0;
|
||||
let truncated = false;
|
||||
outer: for (;;) {
|
||||
const pages = await engine.listPages({
|
||||
limit: PAGE,
|
||||
offset,
|
||||
sort: 'updated_desc',
|
||||
...(opts.sourceIds && opts.sourceIds.length > 0
|
||||
? { sourceIds: opts.sourceIds }
|
||||
: opts.sourceId ? { sourceId: opts.sourceId } : {}),
|
||||
});
|
||||
if (pages.length === 0) break;
|
||||
for (const p of pages) {
|
||||
if (opts.maxScan !== undefined && scanned >= opts.maxScan) { truncated = true; break outer; }
|
||||
scanned += 1;
|
||||
const r = rowFor(p);
|
||||
if (!r) continue;
|
||||
if (r.marker === 'content_flag' && !includeFlagged) continue;
|
||||
if (r.marker === 'content_flag' && !opts.includeFlagged) continue;
|
||||
rows.push(r);
|
||||
if (opts.limit !== undefined && rows.length >= opts.limit) { truncated = true; break outer; }
|
||||
}
|
||||
if (pages.length < PAGE) break;
|
||||
offset += PAGE;
|
||||
}
|
||||
return { rows, scanned, truncated };
|
||||
}
|
||||
|
||||
async function runList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const includeFlagged = args.includes('--include-flagged');
|
||||
const { rows } = await collectQuarantineRows(engine, { includeFlagged });
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({ schema_version: 1, count: rows.length, rows }, null, 2));
|
||||
@@ -94,15 +162,42 @@ async function runClear(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const force = args.includes('--force');
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
// First non-flag positional after the subcommand is the slug.
|
||||
const slug = args.find((a) => !a.startsWith('--'));
|
||||
const srcIdx = args.indexOf('--source-id');
|
||||
const sourceIdFlag = srcIdx >= 0 && args[srcIdx + 1] && !args[srcIdx + 1].startsWith('--')
|
||||
? args[srcIdx + 1]
|
||||
: undefined;
|
||||
// First non-flag positional after the subcommand is the slug (skip the
|
||||
// --source-id value so it can't be mistaken for the slug).
|
||||
const slug = args.find((a, i) => !a.startsWith('--') && !(srcIdx >= 0 && i === srcIdx + 1));
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain quarantine clear <slug> [--force] [--no-embed]');
|
||||
console.error('Usage: gbrain quarantine clear <slug> [--source-id <id>] [--force] [--no-embed]');
|
||||
process.exit(2);
|
||||
}
|
||||
const page = await engine.getPage(slug);
|
||||
// Deterministic source resolution: an unscoped getPage on a slug that
|
||||
// exists in multiple sources returns an arbitrary row, and the re-import
|
||||
// below writes to WHATEVER source that read happened to hit. Resolve the
|
||||
// candidate sources explicitly; ambiguity is an error, not a coin flip.
|
||||
let sourceId = sourceIdFlag;
|
||||
if (!sourceId) {
|
||||
const rows = await engine.executeRaw<{ source_id: string }>(
|
||||
`SELECT source_id FROM pages WHERE slug = $1 AND deleted_at IS NULL ORDER BY source_id`,
|
||||
[slug],
|
||||
);
|
||||
if (rows.length > 1) {
|
||||
console.error(
|
||||
`Slug "${slug}" exists in ${rows.length} sources: ${rows.map(r => r.source_id).join(', ')}.\n` +
|
||||
`Pick one with: gbrain quarantine clear ${slug} --source-id <id>`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
sourceId = rows[0]?.source_id;
|
||||
}
|
||||
// sourceId is resolved above whenever ANY row exists; zero candidates means
|
||||
// the page doesn't exist in any source, so the 'default' fallback read
|
||||
// returns null and we error below either way.
|
||||
const page = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!page) {
|
||||
console.error(`No page found for slug "${slug}".`);
|
||||
console.error(`No page found for slug "${slug}"${sourceIdFlag ? ` in source "${sourceIdFlag}"` : ''}.`);
|
||||
process.exit(2);
|
||||
}
|
||||
const fm = { ...((page.frontmatter ?? {}) as Record<string, unknown>) };
|
||||
|
||||
@@ -120,6 +120,7 @@ function printCodeModelNudge(decision: Extract<NudgeDecision, { shouldNudge: tru
|
||||
|
||||
interface CodePageRow {
|
||||
slug: string;
|
||||
source_id: string;
|
||||
compiled_truth: string;
|
||||
frontmatter: Record<string, unknown> | null;
|
||||
}
|
||||
@@ -133,8 +134,13 @@ async function fetchCodePages(
|
||||
// Direct SQL: listPages doesn't expose source_id filtering, and we need
|
||||
// compiled_truth + frontmatter anyway (not just the Page shape).
|
||||
const sourceClause = sourceId ? `AND p.source_id = '${sourceId.replace(/'/g, "''")}'` : '';
|
||||
// source_id is SELECTed so the per-page re-import below targets each row's
|
||||
// OWN source. Pre-fix this iterated all sources' code pages but imported
|
||||
// with the CLI-level sourceId (undefined without --source), which — now
|
||||
// that import reads/writes are default-scoped — would duplicate every
|
||||
// non-default-source code page into 'default' and re-embed it.
|
||||
const rows = await engine.executeRaw<CodePageRow>(
|
||||
`SELECT p.slug, p.compiled_truth, p.frontmatter
|
||||
`SELECT p.slug, p.source_id, p.compiled_truth, p.frontmatter
|
||||
FROM pages p
|
||||
WHERE p.type = 'code' ${sourceClause}
|
||||
ORDER BY p.slug
|
||||
@@ -299,7 +305,10 @@ export async function runReindexCode(
|
||||
const result = await importCodeFile(engine, relPath, row.compiled_truth, {
|
||||
noEmbed: opts.noEmbed,
|
||||
force: opts.force,
|
||||
sourceId: opts.sourceId,
|
||||
// Each page re-imports into its OWN source (row-level), not
|
||||
// the CLI-level default — reindex must be an in-place
|
||||
// rebuild, never a cross-source copy.
|
||||
sourceId: row.source_id,
|
||||
});
|
||||
if (result.status === 'imported') reindexed++;
|
||||
else if (result.status === 'skipped') skipped++;
|
||||
|
||||
+38
-243
@@ -22,114 +22,37 @@
|
||||
* Recommendation engine. Reads stats + brain size + model tier and
|
||||
* prints structured recommendations. --apply mutates config (each
|
||||
* change logged loud + paste-ready revert command at the end).
|
||||
*
|
||||
* The report builders live in core (src/core/search/modes-report.ts,
|
||||
* tune-recommendations.ts, telemetry.ts) and are shared with the
|
||||
* search_modes / search_stats / search_tune MCP ops. This file owns arg
|
||||
* parsing, text rendering, the --reset lane, and the --apply lane
|
||||
* (config mutation stays CLI-only per [CDX-21]).
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import {
|
||||
MODE_BUNDLES,
|
||||
SEARCH_MODES,
|
||||
SEARCH_MODE_KEY,
|
||||
SEARCH_MODE_CONFIG_KEYS,
|
||||
DEFAULT_SEARCH_MODE,
|
||||
isSearchMode,
|
||||
loadSearchModeConfig,
|
||||
resolveSearchMode,
|
||||
attributeKnob,
|
||||
type SearchMode,
|
||||
type ModeBundle,
|
||||
} from '../core/search/mode.ts';
|
||||
import { readSearchStats, telemetryCoverage, TELEMETRY_COVERAGE_CAVEAT } from '../core/search/telemetry.ts';
|
||||
|
||||
const KNOB_DESCRIPTIONS: Record<keyof ModeBundle, string> = {
|
||||
cache_enabled: 'Semantic query cache on/off',
|
||||
cache_similarity_threshold: 'Cosine-similarity floor for cache hits (0..1)',
|
||||
cache_ttl_seconds: 'Per-row cache TTL',
|
||||
intentWeighting: 'Zero-LLM intent classifier weight adjustments',
|
||||
tokenBudget: 'Per-call token-budget cap (undefined = no cap)',
|
||||
expansion: 'LLM multi-query expansion (Haiku call per search)',
|
||||
searchLimit: 'Default `limit` for the operation layer',
|
||||
reranker_enabled: 'Cross-encoder reranker on/off',
|
||||
reranker_model: 'Provider:model for the reranker',
|
||||
reranker_top_n_in: 'Candidates sent to reranker per call',
|
||||
reranker_top_n_out: 'Cap on reranked output (null = no truncate)',
|
||||
reranker_timeout_ms: 'HTTP timeout for the reranker call',
|
||||
floor_ratio: 'Floor-ratio gate for metadata boosts (0..1, undefined = off)',
|
||||
title_boost: 'Title-phrase boost multiplier (query is a title token-run; 1.0 = off)',
|
||||
// v0.36 cross-modal knobs (D3 registry)
|
||||
cross_modal_both_text_weight: "D6 'both'-mode RRF weight for text branch (0.6 default)",
|
||||
cross_modal_both_image_weight: "D6 'both'-mode RRF weight for image branch (0.4 default)",
|
||||
image_query_text_refinement_weight: 'D13 searchByImage text-refinement RRF weight (0.4 default)',
|
||||
image_query_image_refinement_weight: 'D13 searchByImage image branch RRF weight (0.6 default)',
|
||||
unified_multimodal: 'Phase 3 — route all queries through embedding_multimodal column',
|
||||
unified_multimodal_only: 'Phase 3 strict — bypass dual-column fallback when unified is on',
|
||||
cross_modal_llm_intent: 'Commit 4 — Haiku tie-break for ambiguous modality classification',
|
||||
// v0.40.4 graph signals
|
||||
graph_signals: 'Selective graph signals: adjacency hub + cross-source hub + session diversification',
|
||||
// v0.40.3.0 contextual retrieval
|
||||
contextual_retrieval: 'CR tier (none|title|per_chunk_synopsis) — wraps chunks at embed time',
|
||||
contextual_retrieval_disabled: 'Soft kill switch — neutralizes CR wrapping for queries + new embeds',
|
||||
// v0.42.3.0 autocut
|
||||
autocut: 'Score-discontinuity result-sizing (cuts at the rerank-score cliff; no-op without a reranker)',
|
||||
autocut_jump: 'Autocut sensitivity: min normalized score gap that counts as a cliff (0..1, 0.20 default)',
|
||||
// v0.43 relational recall
|
||||
relationalRetrieval: 'Typed-edge relational recall arm (relational queries walk the graph; no-op otherwise)',
|
||||
relational_retrieval_depth: 'Max hops for relational traversal (1..3, 2 default)',
|
||||
};
|
||||
|
||||
interface SearchModesReport {
|
||||
schema_version: 2;
|
||||
active_mode: SearchMode;
|
||||
active_mode_valid: boolean;
|
||||
resolved: Record<keyof ModeBundle, { value: unknown; source: string; source_detail: string; description: string }>;
|
||||
bundles: Record<SearchMode, ModeBundle>;
|
||||
config_keys: ReadonlyArray<string>;
|
||||
_meta?: {
|
||||
metric_glossary?: Record<string, string>;
|
||||
};
|
||||
}
|
||||
|
||||
async function buildModesReport(engine: BrainEngine): Promise<SearchModesReport> {
|
||||
const input = await loadSearchModeConfig(engine);
|
||||
const resolved = resolveSearchMode(input);
|
||||
|
||||
const knobs: Array<keyof ModeBundle> = [
|
||||
'cache_enabled',
|
||||
'cache_similarity_threshold',
|
||||
'cache_ttl_seconds',
|
||||
'intentWeighting',
|
||||
'tokenBudget',
|
||||
'expansion',
|
||||
'searchLimit',
|
||||
// v0.35.6.0 — floor-ratio surfaced in `gbrain search modes` dashboard
|
||||
// so config drift is legible. Default undefined renders as 'undefined'
|
||||
// in the bundle column, 'mode' source when unset by config/per-call.
|
||||
'floor_ratio',
|
||||
];
|
||||
|
||||
const attributions = {} as SearchModesReport['resolved'];
|
||||
for (const k of knobs) {
|
||||
const a = attributeKnob(k, input, resolved);
|
||||
attributions[k] = {
|
||||
value: a.value,
|
||||
source: a.source,
|
||||
source_detail: a.source_detail,
|
||||
description: KNOB_DESCRIPTIONS[k],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
schema_version: 2,
|
||||
active_mode: resolved.resolved_mode,
|
||||
active_mode_valid: resolved.mode_valid,
|
||||
resolved: attributions,
|
||||
bundles: {
|
||||
conservative: { ...MODE_BUNDLES.conservative },
|
||||
balanced: { ...MODE_BUNDLES.balanced },
|
||||
tokenmax: { ...MODE_BUNDLES.tokenmax },
|
||||
},
|
||||
config_keys: SEARCH_MODE_CONFIG_KEYS,
|
||||
};
|
||||
}
|
||||
import {
|
||||
readSearchStats,
|
||||
readGraphSignalsStats,
|
||||
telemetryCoverage,
|
||||
TELEMETRY_COVERAGE_CAVEAT,
|
||||
type GraphSignalsStatsSection,
|
||||
} from '../core/search/telemetry.ts';
|
||||
import {
|
||||
buildModesReport,
|
||||
KNOB_DESCRIPTIONS,
|
||||
type SearchModesReport,
|
||||
} from '../core/search/modes-report.ts';
|
||||
import {
|
||||
buildTuneRecommendations,
|
||||
TUNE_MIN_CALLS,
|
||||
type TuneRecommendation,
|
||||
} from '../core/search/tune-recommendations.ts';
|
||||
|
||||
function formatModesText(report: SearchModesReport): string {
|
||||
const lines: string[] = [];
|
||||
@@ -210,15 +133,8 @@ async function runStatsSubcommand(engine: BrainEngine, args: string[]): Promise<
|
||||
|
||||
const stats = await readSearchStats(engine, { days: Number.isFinite(days) ? days : 7 });
|
||||
|
||||
// v0.40.4 — graph_signals section. Sourced from:
|
||||
// 1. config: search.graph_signals (or mode bundle default) for the
|
||||
// on/off status.
|
||||
// 2. JSONL audit: graph-signals-failures-*.jsonl for the error count.
|
||||
//
|
||||
// Fire-rate metrics (adjacency_fires, cross_source_fires,
|
||||
// session_demotions) require telemetry table writes from the
|
||||
// applyGraphSignals onMeta callback — wired in a v0.41+ follow-up
|
||||
// (T-todo-2 calibration wave). For now: status + error count.
|
||||
// v0.40.4 — graph_signals section (readGraphSignalsStats now lives in
|
||||
// core/search/telemetry.ts, shared with the search_stats op).
|
||||
const gsSection = await readGraphSignalsStats(engine, Number.isFinite(days) ? days : 7);
|
||||
|
||||
if (json) {
|
||||
@@ -291,57 +207,6 @@ async function runStatsSubcommand(engine: BrainEngine, args: string[]): Promise<
|
||||
printGraphSignalsSection(gsSection);
|
||||
}
|
||||
|
||||
interface GraphSignalsStatsSection {
|
||||
enabled: boolean;
|
||||
source: 'config' | 'mode_default';
|
||||
failures_count: number;
|
||||
/** Failure-reason breakdown across the window (truncated to top reasons). */
|
||||
failures_by_reason: Record<string, number>;
|
||||
}
|
||||
|
||||
async function readGraphSignalsStats(engine: BrainEngine, days: number): Promise<GraphSignalsStatsSection> {
|
||||
// Resolve graph_signals on/off. Mirrors the resolution chain in
|
||||
// src/commands/doctor/checks/graph-embedding.ts:checkGraphSignalsCoverage.
|
||||
// v0.40.4 codex F1: case-insensitive + trim parity with
|
||||
// loadOverridesFromConfig (mode.ts). Without this, search-stats would
|
||||
// silently report the opposite of what the parser actually enables on
|
||||
// values like 'TRUE' or 'True'.
|
||||
const cfg = await engine.getConfig('search.graph_signals').catch(() => null);
|
||||
let enabled: boolean;
|
||||
let source: 'config' | 'mode_default';
|
||||
if (cfg !== null && cfg !== undefined) {
|
||||
const v = cfg.trim().toLowerCase();
|
||||
enabled = v === 'true' || v === '1';
|
||||
source = 'config';
|
||||
} else {
|
||||
const modeRaw = await engine.getConfig('search.mode').catch(() => null);
|
||||
const modeVal = typeof modeRaw === 'string' ? modeRaw.trim().toLowerCase() : '';
|
||||
const mode = modeVal === 'conservative' || modeVal === 'tokenmax' ? modeVal : 'balanced';
|
||||
enabled = mode !== 'conservative';
|
||||
source = 'mode_default';
|
||||
}
|
||||
|
||||
let failures_count = 0;
|
||||
const failures_by_reason: Record<string, number> = {};
|
||||
try {
|
||||
const { readRecentGraphSignalsFailures } = await import('../core/search/graph-signals.ts');
|
||||
const events = readRecentGraphSignalsFailures(days);
|
||||
failures_count = events.length;
|
||||
// The failure event schema has error_summary (not a reason field) —
|
||||
// bucket by the first word of the summary so operators see e.g.
|
||||
// "ECONNREFUSED" / "timeout" / "permission" at a glance.
|
||||
for (const e of events) {
|
||||
const firstWord = (e.error_summary ?? '').split(/[\s:]+/)[0]?.slice(0, 32) || 'unknown';
|
||||
failures_by_reason[firstWord] = (failures_by_reason[firstWord] ?? 0) + 1;
|
||||
}
|
||||
} catch {
|
||||
// Audit reader is best-effort. Missing module / corrupt files →
|
||||
// count stays 0, search-stats still renders.
|
||||
}
|
||||
|
||||
return { enabled, source, failures_count, failures_by_reason };
|
||||
}
|
||||
|
||||
function printGraphSignalsSection(gs: GraphSignalsStatsSection): void {
|
||||
console.log(' Graph signals:');
|
||||
const sourceLabel = gs.source === 'config' ? 'config override' : 'mode default';
|
||||
@@ -362,100 +227,41 @@ function printGraphSignalsSection(gs: GraphSignalsStatsSection): void {
|
||||
}
|
||||
}
|
||||
|
||||
interface TuneRecommendation {
|
||||
knob: string;
|
||||
current: unknown;
|
||||
suggested: unknown;
|
||||
reason: string;
|
||||
apply_command: string;
|
||||
}
|
||||
|
||||
async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const apply = args.includes('--apply');
|
||||
|
||||
const modeInput = await loadSearchModeConfig(engine);
|
||||
const resolved = resolveSearchMode(modeInput);
|
||||
const stats = await readSearchStats(engine, { days: 7 });
|
||||
const report = await buildTuneRecommendations(engine);
|
||||
const recs = report.recommendations;
|
||||
|
||||
const recs: TuneRecommendation[] = [];
|
||||
|
||||
// Recommendation 1: low call volume → no data yet.
|
||||
if (stats.total_calls < 20) {
|
||||
// Recommendation gate: low call volume → no data yet.
|
||||
if (report.status === 'insufficient_data') {
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 2,
|
||||
status: 'insufficient_data',
|
||||
total_calls: stats.total_calls,
|
||||
coverage: telemetryCoverage(),
|
||||
total_calls: report.total_calls,
|
||||
coverage: report.coverage,
|
||||
recommendations: [],
|
||||
message: 'Not enough search activity in the last 7 days to tune. Run `gbrain search stats` after some real usage.',
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
console.log('Not enough search activity in the last 7 days to tune.');
|
||||
console.log(`Total searches: ${stats.total_calls} (need >= 20 for confident recommendations).`);
|
||||
console.log(`Total searches: ${report.total_calls} (need >= ${TUNE_MIN_CALLS} for confident recommendations).`);
|
||||
console.log(`(${TELEMETRY_COVERAGE_CAVEAT} Low counts can reflect this gap, not just low usage.)`);
|
||||
console.log('Use `gbrain serve` or an MCP session for a while, then re-run `gbrain search tune`.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Recommendation 2: budget pressure under conservative.
|
||||
if (resolved.resolved_mode === 'conservative' && stats.total_calls > 0) {
|
||||
const dropPctPerCall = stats.total_budget_dropped / stats.total_calls;
|
||||
if (dropPctPerCall > 2) {
|
||||
recs.push({
|
||||
knob: 'search.mode',
|
||||
current: 'conservative',
|
||||
suggested: 'balanced',
|
||||
reason: `Avg ${dropPctPerCall.toFixed(1)} results dropped per search by the 4K budget. Consider balanced (12K budget) or raise search.tokenBudget.`,
|
||||
apply_command: 'gbrain config set search.mode balanced',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Recommendation 3: high cache hit rate → bump similarity threshold.
|
||||
if (stats.cache_hit_rate > 0.85 && stats.cache_hits + stats.cache_misses > 50) {
|
||||
recs.push({
|
||||
knob: 'search.cache.similarity_threshold',
|
||||
current: resolved.cache_similarity_threshold,
|
||||
suggested: 0.94,
|
||||
reason: `Cache hit rate is ${(stats.cache_hit_rate * 100).toFixed(1)}%. You can raise similarity threshold to 0.94 for tighter freshness at small recall cost.`,
|
||||
apply_command: 'gbrain config set search.cache.similarity_threshold 0.94',
|
||||
});
|
||||
}
|
||||
|
||||
// Recommendation 4: tokenmax + Haiku subagent.
|
||||
const subagentModel = await engine.getConfig('models.tier.subagent');
|
||||
if (resolved.resolved_mode === 'tokenmax' && subagentModel && /haiku/i.test(subagentModel)) {
|
||||
recs.push({
|
||||
knob: 'search.mode',
|
||||
current: 'tokenmax',
|
||||
suggested: 'balanced',
|
||||
reason: `Subagent tier is Haiku but mode is tokenmax. LLM expansion adds ~50ms + ~1¢ per query. Balanced cuts that cost without losing intent weighting or cache.`,
|
||||
apply_command: 'gbrain config set search.mode balanced',
|
||||
});
|
||||
}
|
||||
|
||||
// Recommendation 5: cache disabled but available — fix the free win.
|
||||
if (!resolved.cache_enabled && stats.total_calls > 5) {
|
||||
recs.push({
|
||||
knob: 'search.cache.enabled',
|
||||
current: false,
|
||||
suggested: true,
|
||||
reason: 'Cache is disabled but mode bundles enable it by default. Cache is a free win (zero LLM cost, big latency drop on repeat queries).',
|
||||
apply_command: 'gbrain config unset search.cache.enabled',
|
||||
});
|
||||
}
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 2,
|
||||
status: recs.length === 0 ? 'no_recommendations' : 'has_recommendations',
|
||||
total_calls: stats.total_calls,
|
||||
cache_hit_rate: stats.cache_hit_rate,
|
||||
active_mode: resolved.resolved_mode,
|
||||
coverage: telemetryCoverage(),
|
||||
status: report.status,
|
||||
total_calls: report.total_calls,
|
||||
cache_hit_rate: report.cache_hit_rate,
|
||||
active_mode: report.active_mode,
|
||||
coverage: report.coverage,
|
||||
recommendations: recs,
|
||||
applied: apply ? recs.map(r => r.apply_command) : [],
|
||||
_meta: {
|
||||
@@ -473,7 +279,7 @@ async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<v
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Search tune (last 7 days, active mode: ${resolved.resolved_mode}):`);
|
||||
console.log(`Search tune (last 7 days, active mode: ${report.active_mode}):`);
|
||||
console.log(`(${TELEMETRY_COVERAGE_CAVEAT})`);
|
||||
console.log('');
|
||||
|
||||
@@ -572,20 +378,9 @@ export async function runSearch(engine: BrainEngine, args: string[]): Promise<vo
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `gbrain search modes` is read-only — no DB connection strictly required
|
||||
* for the bundle display IF the engine is given. The dispatch in cli.ts
|
||||
* adds 'search' to its dispatch table so the engine connects normally;
|
||||
* this export is here so future no-engine modes (e.g. `gbrain search --help`
|
||||
* without an engine) could route through it cleanly.
|
||||
*/
|
||||
export const _exports_for_test = {
|
||||
buildModesReport,
|
||||
formatModesText,
|
||||
maybeApplyRecommendation,
|
||||
buildRevertCommand,
|
||||
};
|
||||
|
||||
// Suppress unused-export TS warning — these are intentionally retained for
|
||||
// downstream callers (cli.ts dispatch / future skill linkage).
|
||||
void DEFAULT_SEARCH_MODE;
|
||||
|
||||
+157
-5
@@ -6,6 +6,8 @@ import { importFile } from '../core/import-file.ts';
|
||||
import { collectSyncableFiles } from './import.ts';
|
||||
import {
|
||||
isSyncable,
|
||||
isPoisonedPath,
|
||||
sanitizePathForDisplay,
|
||||
unsyncableReason,
|
||||
matchesAnyGlob,
|
||||
resolveSlugForPath,
|
||||
@@ -220,6 +222,19 @@ export interface SyncResult {
|
||||
embedded: number;
|
||||
pagesAffected: string[];
|
||||
failedFiles?: number; // count of parse failures (Bug 9)
|
||||
/**
|
||||
* Files skipped because their FILENAME contains bracket/control characters
|
||||
* (SyncableReason 'malformed-path'). Informational — these never gate
|
||||
* bookmark advancement; rename the files to import them.
|
||||
*/
|
||||
malformedSkipped?: number;
|
||||
/**
|
||||
* Aggregated alias/undeclared explicit-type warnings (schema.type_warnings,
|
||||
* default on) — one entry per distinct non-canonical type this run.
|
||||
* Carried on the RESULT (not just stderr) so worker-driven syncs surface it
|
||||
* in job results where daemon stderr is invisible.
|
||||
*/
|
||||
type_warnings?: Array<{ kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string; count: number }>;
|
||||
/**
|
||||
* v0.41.13.0 partial-sync fields (only set when status === 'partial').
|
||||
*
|
||||
@@ -557,7 +572,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// importFile call below. Codex perf finding #7: per-file loadActivePack adds
|
||||
// disk/YAML/hash overhead × thousands of files. Best-effort: pack load
|
||||
// failure falls through to legacy inferType (parity preserved).
|
||||
let syncActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
|
||||
let syncActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> } | undefined;
|
||||
try {
|
||||
// v0.41.37.0 #1569: --no-schema-pack escape hatch. Skip pack load entirely so
|
||||
// no user-supplied pack regex (markdown.ts subtype path_pattern) runs during
|
||||
@@ -1119,18 +1134,43 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// old page's backing file is gone from this source's slice of the repo.
|
||||
const renamedToUnsyncable = manifest.renamed
|
||||
.filter(r => inScope(r.from) && isSyncable(r.from, syncOpts) &&
|
||||
!(inScope(r.to) && isSyncable(r.to, syncOpts)))
|
||||
!(inScope(r.to) && isSyncable(r.to, syncOpts)) &&
|
||||
// A rename onto a NON-poison malformed destination (`foo.md` →
|
||||
// `notes [draft].md`) keeps the old row: the content still exists on
|
||||
// disk under the new name, it just can't re-import until renamed —
|
||||
// deleting the row here would be the rename-lane variant of the
|
||||
// reconcile data-loss class (codex re-review P1). Poisoned
|
||||
// destinations (`](`/control chars) still sweep.
|
||||
!(unsyncableReason(r.to, syncOpts) === 'malformed-path' && !isPoisonedPath(r.to)))
|
||||
.map(r => r.from);
|
||||
const filtered: SyncManifest = {
|
||||
added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
deleted: unique([
|
||||
...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)),
|
||||
// 'malformed-path' deletions MUST still process: the classifier makes
|
||||
// junk filenames unsyncable, but their previously-ingested DB rows are
|
||||
// exactly what a delete event is supposed to remove — filtering them
|
||||
// out here would orphan those rows (searchable forever). Mirror of the
|
||||
// metafile carve-out, in the opposite direction.
|
||||
...manifest.deleted.filter(p => inScope(p) &&
|
||||
(isSyncable(p, syncOpts) || unsyncableReason(p, syncOpts) === 'malformed-path')),
|
||||
...renamedToUnsyncable,
|
||||
]),
|
||||
renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
|
||||
};
|
||||
|
||||
// Surface malformed-filename skips: they were silently dropped from the
|
||||
// `filtered` manifest above, and a skip nobody can see reads as "synced".
|
||||
// Rename DESTINATIONS count too (the rename lane keeps the old row for
|
||||
// non-poison destinations, but the new name still can't import).
|
||||
const malformedSkipped = unique([
|
||||
...[...manifest.added, ...manifest.modified]
|
||||
.filter(p => inScope(p) && unsyncableReason(p, syncOpts) === 'malformed-path'),
|
||||
...manifest.renamed
|
||||
.filter(r => inScope(r.to) && unsyncableReason(r.to, syncOpts) === 'malformed-path')
|
||||
.map(r => r.to),
|
||||
]);
|
||||
|
||||
// NAV-4: warn when --exclude filtered out every candidate change — almost
|
||||
// always a mistyped pattern, and otherwise indistinguishable from
|
||||
// "up to date" in the output.
|
||||
@@ -1155,9 +1195,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (filtered.modified.length) slog(` Modified: ${filtered.modified.join(', ')}`);
|
||||
if (filtered.deleted.length) slog(` Deleted: ${filtered.deleted.join(', ')}`);
|
||||
if (filtered.renamed.length) slog(` Renamed: ${filtered.renamed.map(r => `${r.from} -> ${r.to}`).join(', ')}`);
|
||||
if (malformedSkipped.length) {
|
||||
slog(` Skipped (malformed filename — brackets/control chars; rename to import): ${malformedSkipped.map(sanitizePathForDisplay).join(', ')}`);
|
||||
}
|
||||
if (totalChanges === 0) slog(` No syncable changes.`);
|
||||
return {
|
||||
status: 'dry_run',
|
||||
malformedSkipped: malformedSkipped.length,
|
||||
fromCommit: lastCommit,
|
||||
toCommit: headCommit,
|
||||
added: filtered.added.length,
|
||||
@@ -1207,6 +1251,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// pages every time their materialized file landed in a commit.
|
||||
const reason = unsyncableReason(path, syncOpts);
|
||||
if (reason === 'metafile' || reason === 'pruned-dir') continue;
|
||||
// Bare-bracket markdown (pre-gate imports like `notes [draft].md`) keeps
|
||||
// its row — only the poison signature (`](`/control chars) is sweepable.
|
||||
// Deleting a legit page's row while its file sits on disk is data loss.
|
||||
if (reason === 'malformed-path' && !isPoisonedPath(path)) continue;
|
||||
const slug = await resolveSlugByPathOrSourcePath(engine, path, opts.sourceId);
|
||||
try {
|
||||
const existing = await engine.getPage(slug, pageOpts);
|
||||
@@ -1248,6 +1296,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
|
||||
await clearOpCheckpoint(engine, ckpt.paths);
|
||||
await clearOpCheckpoint(engine, ckpt.target);
|
||||
// A commit whose ONLY changes are malformed filenames lands here with
|
||||
// totalChanges === 0 — the anchor advances past those files forever, so
|
||||
// this early return must surface the skips too (structured-review P2).
|
||||
if (malformedSkipped.length > 0) {
|
||||
serr(
|
||||
` ${malformedSkipped.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
malformedSkipped.map(sanitizePathForDisplay).join(', '),
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: 'up_to_date',
|
||||
fromCommit: lastCommit,
|
||||
@@ -1256,6 +1314,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
chunksCreated: 0,
|
||||
embedded: 0,
|
||||
pagesAffected: [],
|
||||
...(malformedSkipped.length > 0 ? { malformedSkipped: malformedSkipped.length } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1429,6 +1488,23 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// advancement at the bottom of this function.
|
||||
const failedFiles: Array<{ path: string; error: string; line?: number }> = [];
|
||||
|
||||
// Alias-footgun visibility (schema.type_warnings, default on): aggregate
|
||||
// per-file type_warning results ONCE per distinct type per run — an
|
||||
// N-thousand-file sync must warn in O(distinct types) lines, not O(files).
|
||||
const typeWarningCounts = new Map<string, import('../core/schema-pack/type-usage.ts').TypeWarningCount>();
|
||||
const noteTypeWarning = (w: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string } | undefined): void => {
|
||||
if (!w) return;
|
||||
const key = `${w.kind}\t${w.type}`;
|
||||
const cur = typeWarningCounts.get(key);
|
||||
if (cur) cur.count++;
|
||||
else typeWarningCounts.set(key, { ...w, count: 1 });
|
||||
};
|
||||
let typeWarningsEnabled = true;
|
||||
try {
|
||||
const v = await engine.getConfig('schema.type_warnings');
|
||||
typeWarningsEnabled = !(v === 'false' || v === '0' || v === 'off');
|
||||
} catch { /* config unavailable → default on */ }
|
||||
|
||||
// v0.18.0+ multi-source: scope deletePage so we only delete the source-A
|
||||
// row, not every same-slug row across all sources.
|
||||
const deleteOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
@@ -1656,8 +1732,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
try {
|
||||
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
|
||||
importResult = result;
|
||||
noteTypeWarning(result.type_warning);
|
||||
if (result.status === 'imported') chunksCreated += result.chunks;
|
||||
else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
else if (result.status === 'skipped' && result.skip_reason === 'malformed_path') {
|
||||
// Informational skip — a bracket/control-char filename can never
|
||||
// import; counting it as a failure would gate the bookmark forever.
|
||||
serr(` Skipped (malformed filename): ${sanitizePathForDisplay(to)}`);
|
||||
} else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
failedFiles.push({ path: to, error: String((result as { error?: string }).error) });
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
@@ -1922,6 +2003,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// duplicate rows that crashed bare-slug subqueries with Postgres 21000.
|
||||
const result = await observed(pacer, () =>
|
||||
importFile(eng, filePath, path, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack }));
|
||||
noteTypeWarning(result.type_warning);
|
||||
if (result.status === 'imported') {
|
||||
chunksCreated += result.chunks;
|
||||
pagesAffected.push(result.slug);
|
||||
@@ -1935,6 +2017,12 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
filesImported++;
|
||||
// v0.42.x (#1794): checkpoint this path so a kill banks it.
|
||||
await markCompleted(path);
|
||||
} else if (result.status === 'skipped' && result.skip_reason === 'malformed_path') {
|
||||
// Informational skip (bracket/control-char filename): never a
|
||||
// failure, and stable across runs — checkpoint it as done so a
|
||||
// resumed sync doesn't re-attempt it forever.
|
||||
serr(` Skipped (malformed filename — rename to import): ${sanitizePathForDisplay(path)}`);
|
||||
await markCompleted(path);
|
||||
} else if (result.status === 'skipped' && (result as any).error) {
|
||||
failedFiles.push({ path, error: String((result as any).error) });
|
||||
} else {
|
||||
@@ -2444,6 +2532,20 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
slog(`Text imported. Run 'gbrain embed --stale' to generate embeddings.`);
|
||||
}
|
||||
|
||||
if (malformedSkipped.length > 0) {
|
||||
serr(
|
||||
`\n ${malformedSkipped.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars) — rename to import. Not counted as failures.`,
|
||||
);
|
||||
}
|
||||
|
||||
const typeWarnings = [...typeWarningCounts.values()];
|
||||
if (typeWarningsEnabled && typeWarnings.length > 0) {
|
||||
const { renderTypeWarningSummary } = await import('../core/schema-pack/type-usage.ts');
|
||||
for (const line of renderTypeWarningSummary(typeWarnings)) serr(` ${line}`);
|
||||
serr(` (silence with: gbrain config set schema.type_warnings false)`);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'synced',
|
||||
fromCommit: lastCommit,
|
||||
@@ -2455,6 +2557,8 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
chunksCreated,
|
||||
embedded,
|
||||
pagesAffected,
|
||||
malformedSkipped: malformedSkipped.length,
|
||||
...(typeWarningsEnabled && typeWarnings.length > 0 ? { type_warnings: typeWarnings } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2484,9 +2588,11 @@ async function performFullSync(
|
||||
// code --dry-run` always reported zero files even when ~1500 code
|
||||
// files were waiting.
|
||||
if (opts.dryRun) {
|
||||
const dryRunMalformed: string[] = [];
|
||||
let allFiles = collectSyncableFiles(syncScopeRoot, {
|
||||
strategy: opts.strategy ?? 'markdown',
|
||||
includeGitignored: opts.includeGitignored,
|
||||
onExcluded: (rel) => { dryRunMalformed.push(rel); },
|
||||
});
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
|
||||
@@ -2496,6 +2602,14 @@ async function performFullSync(
|
||||
`${allFiles.length} file(s) would be imported ` +
|
||||
`from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`,
|
||||
);
|
||||
if (dryRunMalformed.length > 0) {
|
||||
slog(
|
||||
` ${dryRunMalformed.length} file(s) would be skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
dryRunMalformed.slice(0, 20).map(sanitizePathForDisplay).join(', ') +
|
||||
(dryRunMalformed.length > 20 ? `, … (+${dryRunMalformed.length - 20} more)` : ''),
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: 'dry_run',
|
||||
fromCommit: null,
|
||||
@@ -2662,10 +2776,23 @@ async function performFullSync(
|
||||
// root-level sync of this source) are out of this walk's sight and must
|
||||
// not be treated as stale.
|
||||
const scopePrefix = slugRoot ? relative(gitContextRoot, syncScopeRoot) + '/' : '';
|
||||
// 'malformed-path' rows ARE reconcile-eligible: junk filenames (bracket /
|
||||
// control-char paths minted by misbehaving producers) can never be
|
||||
// re-imported, so their rows are permanent search pollution unless the
|
||||
// reconcile can sweep them. Strategy safety is preserved by classifier
|
||||
// ordering — a path that fails the strategy check classifies as
|
||||
// 'strategy', never 'malformed-path', so a markdown sync still can't
|
||||
// delete code pages. The #1433 metafile protection is likewise untouched.
|
||||
const reconcileEligible = (p: string): boolean =>
|
||||
isSyncable(p, reconcileSyncOpts) ||
|
||||
// Only the poison signature is sweepable; bare-bracket markdown rows
|
||||
// from pre-gate releases survive reconcile (their file still exists —
|
||||
// deleting the row would be silent data loss; cross-model finding).
|
||||
(unsyncableReason(p, reconcileSyncOpts) === 'malformed-path' && isPoisonedPath(p));
|
||||
const plan = planReconcileDeletes(
|
||||
rows,
|
||||
currentFiles,
|
||||
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && isSyncable(p, reconcileSyncOpts),
|
||||
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && reconcileEligible(p),
|
||||
);
|
||||
if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) {
|
||||
// #2828 mass-delete safety valve: a reconcile that would sweep more than
|
||||
@@ -2722,6 +2849,15 @@ async function performFullSync(
|
||||
);
|
||||
}
|
||||
const deleteScopedOpts = { sourceId: sid };
|
||||
// Malformed-path rows get their own line: unlike genuinely-deleted
|
||||
// files, THEIR backing file is usually still on disk (the walker
|
||||
// excludes it), so "source file was removed" would be a lie and the
|
||||
// rename-to-rescue path must be stated at the moment of removal, not
|
||||
// only in a doctor check the operator may see later (red-team catch).
|
||||
const malformedDeleted = deletableSlugs.filter(slug => {
|
||||
const sp = pathBySlug.get(slug);
|
||||
return sp != null && unsyncableReason(sp, reconcileSyncOpts) === 'malformed-path';
|
||||
}).length;
|
||||
for (let i = 0; i < deletableSlugs.length; i += DELETE_BATCH_SIZE) {
|
||||
const batch = deletableSlugs.slice(i, i + DELETE_BATCH_SIZE);
|
||||
try {
|
||||
@@ -2738,6 +2874,12 @@ async function performFullSync(
|
||||
}
|
||||
if (reconciledDeletes > 0) {
|
||||
slog(` Reconciled ${reconciledDeletes} stale page(s) whose source file was removed.`);
|
||||
if (malformedDeleted > 0) {
|
||||
slog(
|
||||
` (${malformedDeleted} of them had malformed bracket/control-char filenames — ` +
|
||||
`their files may still exist on disk; rename a file to re-import its content.)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2774,6 +2916,11 @@ async function performFullSync(
|
||||
chunksCreated: result.chunksCreated,
|
||||
embedded,
|
||||
pagesAffected: [],
|
||||
// Warning aggregates ride the result for worker/JSON consumers — a full
|
||||
// sync that only prints to a daemon's stderr hides them from cron
|
||||
// topologies (codex re-review; same rationale as the incremental path).
|
||||
...(result.malformedSkipped ? { malformedSkipped: result.malformedSkipped } : {}),
|
||||
...(result.type_warnings ? { type_warnings: result.type_warnings } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3497,6 +3644,11 @@ See also:
|
||||
deleted: r.result.deleted,
|
||||
chunks_created: r.result.chunksCreated,
|
||||
embedded: r.result.embedded,
|
||||
// Warning aggregates (malformed filenames, alias/undeclared
|
||||
// types) — the whole point of the result-field plumbing is that
|
||||
// JSON/worker consumers can see them (codex re-review).
|
||||
...(r.result.malformedSkipped ? { malformed_skipped: r.result.malformedSkipped } : {}),
|
||||
...(r.result.type_warnings ? { type_warnings: r.result.type_warnings } : {}),
|
||||
} : {}),
|
||||
...(r.error ? { error: r.error } : {}),
|
||||
}));
|
||||
|
||||
+87
-179
@@ -10,25 +10,21 @@
|
||||
* takes supersede <slug> --row N ... — strikethrough old + append new
|
||||
* takes resolve <slug> --row N --outcome true|false [--value N --unit u]
|
||||
*
|
||||
* Markdown is canonical. Every mutate command:
|
||||
* 1. acquires the per-page file lock
|
||||
* 2. re-reads the .md file
|
||||
* 3. applies the edit via takes-fence (upsertTakeRow / supersedeRow)
|
||||
* 4. writes the .md file back
|
||||
* 5. mirrors to the DB via the engine method
|
||||
* 6. releases the lock (auto via withPageLock)
|
||||
* Markdown is canonical. Every mutate command routes through the shared
|
||||
* write-through core (src/core/takes-write.ts — also the takes_* MCP ops'
|
||||
* backend): lock → resolve page → fence edit → write .md → DB mirror. This
|
||||
* file owns arg parsing + rendering + exit codes only.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { existsSync } from 'node:fs';
|
||||
import type { BrainEngine, TakeKind } from '../core/engine.ts';
|
||||
import {
|
||||
parseTakesFence,
|
||||
upsertTakeRow,
|
||||
supersedeRow,
|
||||
type ParsedTake,
|
||||
} from '../core/takes-fence.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
addTakeToPage,
|
||||
updateTakeOnPage,
|
||||
supersedeTakeOnPage,
|
||||
resolveTakeOnPage,
|
||||
TakesWriteError,
|
||||
} from '../core/takes-write.ts';
|
||||
import { resolveSourceId } from '../core/source-resolver.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
@@ -60,8 +56,22 @@ async function resolveBrainDir(engine: BrainEngine | null, explicitDir: string |
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function pageFilePath(brainDir: string, slug: string): string {
|
||||
return join(brainDir, `${slug}.md`);
|
||||
/**
|
||||
* Map a TakesWriteError to the historical CLI error surface (stderr + exit 1).
|
||||
* Message text preserves the pre-extraction wording users and scripts saw.
|
||||
*/
|
||||
function exitTakesError(err: unknown): never {
|
||||
if (err instanceof TakesWriteError) {
|
||||
switch (err.code) {
|
||||
case 'page_not_found':
|
||||
console.error(`${err.message} Run \`gbrain sync\` first.`);
|
||||
process.exit(1);
|
||||
default:
|
||||
console.error(err.hint && err.code !== 'holder_denied' ? `${err.message} ${err.hint}` : err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
function ensureKind(raw: string | undefined): TakeKind {
|
||||
@@ -86,23 +96,6 @@ function ensureFloat(raw: string | undefined, fallback: number): number {
|
||||
return n;
|
||||
}
|
||||
|
||||
async function getPageId(engine: BrainEngine, slug: string, sourceId?: string): Promise<number> {
|
||||
const rows = sourceId
|
||||
? await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = $1 AND source_id = $2 LIMIT 1`,
|
||||
[slug, sourceId],
|
||||
)
|
||||
: await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = $1 LIMIT 1`,
|
||||
[slug],
|
||||
);
|
||||
if (!rows[0]) {
|
||||
console.error(`Page not found in brain: ${slug}${sourceId ? ` (source=${sourceId})` : ''}. Run \`gbrain sync\` first.`);
|
||||
process.exit(1);
|
||||
}
|
||||
return rows[0].id;
|
||||
}
|
||||
|
||||
// Fail-closed (#2698 residual, TODOS.md): `resolveSourceId` only ever
|
||||
// throws when a source WAS explicitly in play — an invalid or
|
||||
// unregistered `GBRAIN_SOURCE`, a `.gbrain-source` dotfile pointing at a
|
||||
@@ -117,16 +110,6 @@ async function resolveTakesSourceId(engine: BrainEngine): Promise<string> {
|
||||
return resolveSourceId(engine, null);
|
||||
}
|
||||
|
||||
function readBodyOrEmpty(path: string): string {
|
||||
if (!existsSync(path)) return '';
|
||||
return readFileSync(path, 'utf-8');
|
||||
}
|
||||
|
||||
function writeBody(path: string, body: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, body, 'utf-8');
|
||||
}
|
||||
|
||||
// --- Subcommands ---
|
||||
|
||||
async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
@@ -209,27 +192,15 @@ async function cmdAdd(engine: BrainEngine, args: string[], sourceId?: string): P
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
// Resolve the page BEFORE touching the markdown. getPageId exits 1 when the
|
||||
// page isn't in the brain; doing this after writeBody left a .md file
|
||||
// carrying a take with no DB row — invisible to scorecard/calibration but
|
||||
// present on disk, so a later `takes add` would number the next row past a
|
||||
// take the DB never saw. update/supersede/resolve already resolve first.
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
const { body: nextBody, rowNum } = upsertTakeRow(body, {
|
||||
claim, kind, holder, weight, source, sinceDate: since, active: true,
|
||||
});
|
||||
writeBody(path, nextBody);
|
||||
|
||||
await engine.addTakesBatch([{
|
||||
page_id: pageId, row_num: rowNum, claim, kind, holder, weight,
|
||||
since_date: since, source, active: true, superseded_by: null,
|
||||
}]);
|
||||
try {
|
||||
const { rowNum } = await addTakeToPage(
|
||||
{ engine, slug, brainDir, sourceId },
|
||||
{ claim, kind, holder, weight, source, sinceDate: since },
|
||||
);
|
||||
console.log(`Added take #${rowNum} to ${slug}.`);
|
||||
});
|
||||
} catch (err) {
|
||||
exitTakesError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function cmdUpdate(engine: BrainEngine, args: string[], sourceId?: string): Promise<void> {
|
||||
@@ -250,36 +221,20 @@ async function cmdUpdate(engine: BrainEngine, args: string[], sourceId?: string)
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
await engine.updateTake(pageId, rowNum, fields);
|
||||
|
||||
// Sync the markdown table: read fence, find row, apply field updates, re-render.
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
const parsed = parseTakesFence(body);
|
||||
const target = parsed.takes.find(t => t.rowNum === rowNum);
|
||||
if (!target) {
|
||||
console.warn(`[takes update] DB updated but row #${rowNum} not in markdown fence on disk; markdown may be out of sync. Run 'gbrain extract takes --slugs ${slug}' to reconcile.`);
|
||||
return;
|
||||
}
|
||||
const updated: ParsedTake = {
|
||||
...target,
|
||||
weight: fields.weight ?? target.weight,
|
||||
source: fields.source ?? target.source,
|
||||
sinceDate: fields.since_date ?? target.sinceDate,
|
||||
};
|
||||
// Replace the row in-place by stripping the fence and re-rendering all rows.
|
||||
const allRows = parsed.takes.map(t => t.rowNum === rowNum ? updated : t);
|
||||
// Round-trip via upsertTakeRow with no new row: easiest is to render manually.
|
||||
const { renderTakesFence, TAKES_FENCE_BEGIN, TAKES_FENCE_END } = await import('../core/takes-fence.ts');
|
||||
const newFence = renderTakesFence(allRows);
|
||||
const beginIdx = body.indexOf(TAKES_FENCE_BEGIN);
|
||||
const endIdx = body.indexOf(TAKES_FENCE_END, beginIdx + TAKES_FENCE_BEGIN.length);
|
||||
const out = body.slice(0, beginIdx) + newFence + body.slice(endIdx + TAKES_FENCE_END.length);
|
||||
writeBody(path, out);
|
||||
// v0.46.x (EV1): markdown is canonical, so a row missing from the on-disk
|
||||
// fence now REFUSES the whole write instead of the old DB-update-then-warn
|
||||
// path — that path was self-defeating (its own reconcile hint, extract
|
||||
// takes, would clobber the DB-only update it had just written).
|
||||
try {
|
||||
await updateTakeOnPage(
|
||||
{ engine, slug, brainDir, sourceId },
|
||||
rowNum,
|
||||
{ weight: fields.weight, source: fields.source, sinceDate: fields.since_date },
|
||||
);
|
||||
console.log(`Updated take #${rowNum} on ${slug}.`);
|
||||
});
|
||||
} catch (err) {
|
||||
exitTakesError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: string): Promise<void> {
|
||||
@@ -295,39 +250,29 @@ async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: stri
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
// Read existing row to inherit kind/holder unless overridden
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: true, limit: 500 });
|
||||
const target = existing.find(t => t.row_num === rowNum);
|
||||
if (!target) {
|
||||
console.error(`Row #${rowNum} not found on ${slug}.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const kind = ensureKind(flagValue(args, '--kind') ?? target.kind);
|
||||
const holder = flagValue(args, '--who') ?? target.holder;
|
||||
const weight = ensureFloat(flagValue(args, '--weight'), Math.max(0, target.weight - 0.1));
|
||||
const source = flagValue(args, '--source');
|
||||
const since = flagValue(args, '--since');
|
||||
|
||||
const dbResult = await engine.supersedeTake(pageId, rowNum, {
|
||||
claim, kind, holder, weight, source, since_date: since, active: true,
|
||||
});
|
||||
|
||||
// Mirror in markdown
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
if (parseTakesFence(body).takes.find(t => t.rowNum === rowNum)) {
|
||||
const { body: nextBody } = supersedeRow(body, rowNum, {
|
||||
claim, kind, holder, weight, source, sinceDate: since,
|
||||
});
|
||||
writeBody(path, nextBody);
|
||||
} else {
|
||||
console.warn(`[takes supersede] DB updated but markdown lacks row #${rowNum}; only DB written.`);
|
||||
}
|
||||
console.log(`Superseded #${dbResult.oldRow} → new #${dbResult.newRow} on ${slug}.`);
|
||||
});
|
||||
// v0.46.x (EV1): fence-first — kind/holder inherit from the MARKDOWN row
|
||||
// (canonical), the fence assigns the new row number, and a row absent from
|
||||
// the on-disk fence refuses instead of the old DB-only write.
|
||||
const kindArg = flagValue(args, '--kind');
|
||||
try {
|
||||
const result = await supersedeTakeOnPage(
|
||||
{ engine, slug, brainDir, sourceId },
|
||||
rowNum,
|
||||
{
|
||||
claim,
|
||||
kind: kindArg !== undefined ? ensureKind(kindArg) : undefined,
|
||||
holder: flagValue(args, '--who'),
|
||||
weight: flagValue(args, '--weight') !== undefined
|
||||
? ensureFloat(flagValue(args, '--weight'), 0.5)
|
||||
: undefined,
|
||||
source: flagValue(args, '--source'),
|
||||
sinceDate: flagValue(args, '--since'),
|
||||
},
|
||||
);
|
||||
console.log(`Superseded #${result.oldRow} → new #${result.newRow} on ${slug}.`);
|
||||
} catch (err) {
|
||||
exitTakesError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function cmdResolve(engine: BrainEngine, args: string[], sourceId?: string): Promise<void> {
|
||||
@@ -374,61 +319,24 @@ async function cmdResolve(engine: BrainEngine, args: string[], sourceId?: string
|
||||
const source = flagValue(args, '--evidence') ?? flagValue(args, '--source');
|
||||
const resolvedBy = flagValue(args, '--by') ?? resolveOwnerHolder({ configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
await engine.resolveTake(pageId, rowNum, {
|
||||
quality,
|
||||
outcome,
|
||||
value,
|
||||
unit,
|
||||
source,
|
||||
resolvedBy,
|
||||
});
|
||||
|
||||
// Mirror resolution into the markdown fence so the page is self-describing.
|
||||
// The renderer conditionally widens the table to 13 columns when at least one
|
||||
// row has resolution data; pages with no resolved rows keep the 7-col shape.
|
||||
// Round-trip via parseTakesFence + renderTakesFence preserves all rows.
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
await withPageLock(slug, async () => {
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
if (!body) {
|
||||
console.warn(`[takes resolve] markdown file not found at ${path}; DB updated but on-disk page absent.`);
|
||||
return;
|
||||
}
|
||||
const { parseTakesFence, renderTakesFence, TAKES_FENCE_BEGIN, TAKES_FENCE_END } = await import('../core/takes-fence.ts');
|
||||
const parsed = parseTakesFence(body);
|
||||
const target = parsed.takes.find(t => t.rowNum === rowNum);
|
||||
if (!target) {
|
||||
console.warn(`[takes resolve] DB updated but row #${rowNum} not in markdown fence; run 'gbrain extract takes --slugs ${slug}' to reconcile.`);
|
||||
return;
|
||||
}
|
||||
// Derive resolved fields from the inputs. Mirror the engine semantics:
|
||||
// quality wins when both set; partial → outcome=null.
|
||||
const finalQuality = quality ?? (outcome === true ? 'correct' : outcome === false ? 'incorrect' : undefined);
|
||||
if (!finalQuality) return; // unreachable — covered by earlier validation
|
||||
const finalOutcome = finalQuality === 'partial' ? undefined
|
||||
: finalQuality === 'correct' ? true : false;
|
||||
const updated = {
|
||||
...target,
|
||||
resolvedAt: new Date().toISOString().slice(0, 10),
|
||||
resolvedQuality: finalQuality,
|
||||
resolvedOutcome: finalOutcome,
|
||||
resolvedEvidence: source,
|
||||
resolvedValue: value,
|
||||
resolvedUnit: unit,
|
||||
resolvedBy,
|
||||
};
|
||||
const allRows = parsed.takes.map(t => t.rowNum === rowNum ? updated : t);
|
||||
const newFence = renderTakesFence(allRows);
|
||||
const beginIdx = body.indexOf(TAKES_FENCE_BEGIN);
|
||||
const endIdx = body.indexOf(TAKES_FENCE_END, beginIdx + TAKES_FENCE_BEGIN.length);
|
||||
const out = body.slice(0, beginIdx) + newFence + body.slice(endIdx + TAKES_FENCE_END.length);
|
||||
writeBody(path, out);
|
||||
});
|
||||
|
||||
const finalQuality = quality ?? (outcome === true ? 'correct' : outcome === false ? 'incorrect' : 'unknown');
|
||||
// Back-compat --outcome maps onto quality; the shared core takes quality only.
|
||||
const finalQuality = quality ?? (outcome === true ? 'correct' : 'incorrect');
|
||||
|
||||
// v0.46.x (EV1): markdown is canonical — the fence row must exist on disk
|
||||
// (the old path resolved the DB first and warned when the fence lacked the
|
||||
// row, leaving a resolution the next reconcile couldn't see).
|
||||
try {
|
||||
await resolveTakeOnPage(
|
||||
{ engine, slug, brainDir, sourceId },
|
||||
rowNum,
|
||||
{ quality: finalQuality, evidence: source, value, unit, resolvedBy },
|
||||
);
|
||||
} catch (err) {
|
||||
exitTakesError(err);
|
||||
}
|
||||
|
||||
const valueSummary = valueStr ? ` value=${value}${unit ? ` ${unit}` : ''}` : '';
|
||||
console.log(`Resolved take #${rowNum} on ${slug}: quality=${finalQuality}${valueSummary}.`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* CLI→MCP gap-closure wave [OV6] — per-subcommand thin-client routing for the
|
||||
* commands whose reads/writes gained MCP ops: takes (list/search/scorecard/
|
||||
* calibration + the write verbs), search (modes/stats/tune, read-only forms),
|
||||
* jobs stats, cache stats, and quarantine list. The salience/anomalies/
|
||||
* graph-query precedent, engine-free: each routable subcommand maps onto its
|
||||
* op over callRemoteTool; everything else returns false so the caller falls
|
||||
* through to refuseThinClient's pinpoint hint.
|
||||
*
|
||||
* Config-MUTATING forms stay host-side by design: `search modes --reset` and
|
||||
* `search tune --apply` (CDX-21), plus `search modes --source <mode>` (the
|
||||
* reset dry-run — previews what a reset would change on the host), `cache
|
||||
* clear|prune`, `quarantine scan|clear`, `takes extract|revisit`.
|
||||
*/
|
||||
|
||||
import type { GBrainConfig } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
|
||||
function flagValue(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
return i === -1 ? undefined : args[i + 1];
|
||||
}
|
||||
|
||||
function num(v: string | undefined): number | undefined {
|
||||
if (v === undefined) return undefined;
|
||||
const n = parseFloat(v);
|
||||
return Number.isFinite(n) ? n : undefined;
|
||||
}
|
||||
|
||||
async function call(cfg: GBrainConfig, tool: string, args: Record<string, unknown>): Promise<unknown> {
|
||||
return unpackToolResult(await callRemoteTool(cfg, tool, args));
|
||||
}
|
||||
|
||||
function printJson(result: unknown): void {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognized-but-malformed routable subcommand: print the SAME usage string
|
||||
* the host CLI prints (copied verbatim from src/commands/takes.ts) and exit 1.
|
||||
* Returning false here instead would fall through to the host-bound refusal
|
||||
* hint, which misleads (the subcommand IS routable — the args are just wrong).
|
||||
*/
|
||||
function usageExit(...lines: string[]): never {
|
||||
for (const line of lines) console.error(line);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Route a thin-client invocation to its MCP op. Returns true when handled
|
||||
* (output printed); false ONLY when the subcommand is genuinely host-bound
|
||||
* and the caller should refuse with the hint. Routable subcommands with
|
||||
* missing/invalid required args exit 1 with the host CLI's usage string
|
||||
* instead of returning false. Remote op errors propagate (the mcp-client
|
||||
* error surface already names the op + reason).
|
||||
*/
|
||||
export async function routeThinClientCommand(
|
||||
cfg: GBrainConfig,
|
||||
command: string,
|
||||
args: string[],
|
||||
): Promise<boolean> {
|
||||
const sub = args[0];
|
||||
const rest = args.slice(1);
|
||||
|
||||
if (command === 'takes') {
|
||||
switch (sub) {
|
||||
case 'list': {
|
||||
const slug = rest[0] && !rest[0].startsWith('-') ? rest[0] : undefined;
|
||||
printJson(await call(cfg, 'takes_list', {
|
||||
...(slug ? { page_slug: slug } : {}),
|
||||
...(flagValue(rest, '--who') ? { holder: flagValue(rest, '--who') } : {}),
|
||||
...(flagValue(rest, '--kind') ? { kind: flagValue(rest, '--kind') } : {}),
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
case 'search': {
|
||||
if (!rest[0]) {
|
||||
usageExit('Usage: gbrain takes search "<query>" [--who h] [--json]');
|
||||
}
|
||||
printJson(await call(cfg, 'takes_search', { query: rest[0], ...(num(flagValue(rest, '--limit')) !== undefined ? { limit: num(flagValue(rest, '--limit')) } : {}) }));
|
||||
return true;
|
||||
}
|
||||
case 'scorecard': {
|
||||
const holder = rest[0] && !rest[0].startsWith('--') ? rest[0] : flagValue(rest, '--holder');
|
||||
printJson(await call(cfg, 'takes_scorecard', { ...(holder ? { holder } : {}) }));
|
||||
return true;
|
||||
}
|
||||
case 'calibration': {
|
||||
printJson(await call(cfg, 'takes_calibration', { ...(flagValue(rest, '--holder') ? { holder: flagValue(rest, '--holder') } : {}) }));
|
||||
return true;
|
||||
}
|
||||
case 'add': {
|
||||
const slug = rest[0];
|
||||
const claim = flagValue(rest, '--claim');
|
||||
const kind = flagValue(rest, '--kind');
|
||||
const holder = flagValue(rest, '--who');
|
||||
if (!slug || !claim || !kind || !holder) {
|
||||
usageExit('Usage: gbrain takes add <slug> --claim "..." --kind <k> --who <h> [--weight 0.5] [--source "..."] [--since YYYY-MM]');
|
||||
}
|
||||
const res = await call(cfg, 'takes_add', {
|
||||
slug,
|
||||
claim,
|
||||
kind,
|
||||
holder,
|
||||
...(num(flagValue(rest, '--weight')) !== undefined ? { weight: num(flagValue(rest, '--weight')) } : {}),
|
||||
...(flagValue(rest, '--source') ? { source: flagValue(rest, '--source') } : {}),
|
||||
...(flagValue(rest, '--since') ? { since: flagValue(rest, '--since') } : {}),
|
||||
}) as { row_num: number };
|
||||
console.log(`Added take #${res.row_num} to ${slug}. (routed to the brain host)`);
|
||||
return true;
|
||||
}
|
||||
case 'update': {
|
||||
const slug = rest[0];
|
||||
const row = num(flagValue(rest, '--row'));
|
||||
if (!slug || row === undefined) {
|
||||
usageExit('Usage: gbrain takes update <slug> --row N [--weight 0.7] [--source "..."] [--since YYYY-MM]');
|
||||
}
|
||||
await call(cfg, 'takes_update', {
|
||||
slug, row_num: row,
|
||||
...(num(flagValue(rest, '--weight')) !== undefined ? { weight: num(flagValue(rest, '--weight')) } : {}),
|
||||
...(flagValue(rest, '--source') ? { source: flagValue(rest, '--source') } : {}),
|
||||
...(flagValue(rest, '--since') ? { since: flagValue(rest, '--since') } : {}),
|
||||
});
|
||||
console.log(`Updated take #${row} on ${slug}. (routed to the brain host)`);
|
||||
return true;
|
||||
}
|
||||
case 'resolve': {
|
||||
const slug = rest[0];
|
||||
const row = num(flagValue(rest, '--row'));
|
||||
const RESOLVE_USAGE = [
|
||||
'Usage: gbrain takes resolve <slug> --row N --quality correct|incorrect|partial|unresolvable [--evidence "..."] [--value N --unit usd|pct|count] [--by <slug>]',
|
||||
' (back-compat) gbrain takes resolve <slug> --row N --outcome true|false [...]',
|
||||
];
|
||||
// Back-compat outcome lane (mirrors cmdResolve in src/commands/takes.ts):
|
||||
// --quality wins when present; otherwise --outcome true|false maps onto
|
||||
// correct|incorrect. Any other outcome value is a usage error.
|
||||
let quality = flagValue(rest, '--quality');
|
||||
const outcomeStr = flagValue(rest, '--outcome');
|
||||
if (!quality && outcomeStr !== undefined) {
|
||||
if (outcomeStr !== 'true' && outcomeStr !== 'false') {
|
||||
usageExit(...RESOLVE_USAGE);
|
||||
}
|
||||
quality = outcomeStr === 'true' ? 'correct' : 'incorrect';
|
||||
console.error('[deprecated] --outcome is the v0.28 alias for --quality. Prefer --quality correct|incorrect|partial in new scripts.');
|
||||
}
|
||||
if (!slug || row === undefined || !quality) {
|
||||
usageExit(...RESOLVE_USAGE);
|
||||
}
|
||||
const res = await call(cfg, 'takes_resolve', {
|
||||
slug, row_num: row, quality,
|
||||
...(flagValue(rest, '--evidence') ? { evidence: flagValue(rest, '--evidence') } : {}),
|
||||
...(num(flagValue(rest, '--value')) !== undefined ? { value: num(flagValue(rest, '--value')) } : {}),
|
||||
...(flagValue(rest, '--unit') ? { unit: flagValue(rest, '--unit') } : {}),
|
||||
}) as { resolved_by: string };
|
||||
console.log(`Resolved take #${row} on ${slug}: quality=${quality} (as ${res.resolved_by}).`);
|
||||
return true;
|
||||
}
|
||||
case 'supersede': {
|
||||
const slug = rest[0];
|
||||
const row = num(flagValue(rest, '--row'));
|
||||
const claim = flagValue(rest, '--claim');
|
||||
if (!slug || row === undefined || !claim) {
|
||||
usageExit('Usage: gbrain takes supersede <slug> --row N --claim "..." [--kind k] [--who h] [--weight 0.5] [--source "..."]');
|
||||
}
|
||||
const res = await call(cfg, 'takes_supersede', {
|
||||
slug, row_num: row, claim,
|
||||
...(flagValue(rest, '--kind') ? { kind: flagValue(rest, '--kind') } : {}),
|
||||
...(flagValue(rest, '--who') ? { holder: flagValue(rest, '--who') } : {}),
|
||||
...(num(flagValue(rest, '--weight')) !== undefined ? { weight: num(flagValue(rest, '--weight')) } : {}),
|
||||
}) as { old_row: number; new_row: number };
|
||||
console.log(`Superseded #${res.old_row} → new #${res.new_row} on ${slug}. (routed to the brain host)`);
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false; // extract / revisit / unknown — host-bound, refuse with hint
|
||||
}
|
||||
}
|
||||
|
||||
if (command === 'search') {
|
||||
if (sub === 'modes' && !rest.includes('--reset') && !rest.includes('--source')) {
|
||||
printJson(await call(cfg, 'search_modes', {}));
|
||||
return true;
|
||||
}
|
||||
if (sub === 'stats') {
|
||||
printJson(await call(cfg, 'search_stats', { ...(num(flagValue(rest, '--days')) !== undefined ? { days: num(flagValue(rest, '--days')) } : {}) }));
|
||||
return true;
|
||||
}
|
||||
if (sub === 'tune' && !rest.includes('--apply')) {
|
||||
printJson(await call(cfg, 'search_tune', {}));
|
||||
return true;
|
||||
}
|
||||
return false; // modes --reset / modes --source (the reset dry-run) / tune --apply / diagnose — host-side config or live probe
|
||||
}
|
||||
|
||||
if (command === 'jobs' && sub === 'stats') {
|
||||
printJson(await call(cfg, 'get_job_stats', { ...(flagValue(rest, '--queue') ? { queue: flagValue(rest, '--queue') } : {}) }));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (command === 'cache' && sub === 'stats') {
|
||||
printJson(await call(cfg, 'cache_stats', {}));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (command === 'quarantine' && sub === 'list') {
|
||||
printJson(await call(cfg, 'quarantine_list', { ...(rest.includes('--include-flagged') ? { include_flagged: true } : {}) }));
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -463,6 +463,37 @@ export async function runPostUpgrade(args: string[] = []): Promise<void> {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// Waiting-TTL pre-notice (one-shot, warn-before-act). The worker
|
||||
// gates its first sweep behind the SAME flag via runWaitingTtlTick
|
||||
// (notice → grace window → sweep) because daemon restarts never run
|
||||
// this CLI path — this banner is the interactive channel. Stamping
|
||||
// the ISO timestamp here starts the same grace clock, so an operator
|
||||
// who sees this banner gets the full window to tune before anything
|
||||
// is cancelled.
|
||||
try {
|
||||
const { admissionKilled, resolveTtlNames, countTtlExpiredWaiting, ttlNoticeGraceMs, TTL_NOTICE_SHOWN_KEY } =
|
||||
await import('../core/minions/admission.ts');
|
||||
const shown = await engine.getConfig(TTL_NOTICE_SHOWN_KEY);
|
||||
if ((shown == null || shown.trim() === '') && !admissionKilled()) {
|
||||
const ttlNames = await resolveTtlNames(engine);
|
||||
const { total: affected, by_name } = await countTtlExpiredWaiting(engine, ttlNames);
|
||||
const parts = [...ttlNames].map(([name, hours]) => `${name} > ${hours}h: ${by_name[name] ?? 0}`);
|
||||
console.log('');
|
||||
console.log(`⚠ [gbrain] Waiting-TTL is now active: queued jobs that never get claimed are`);
|
||||
console.log(` cancelled after their per-type TTL (${parts.join('; ') || 'defaults'}).`);
|
||||
if (affected > 0) {
|
||||
console.log(` ${affected} currently-queued job(s) already exceed their TTL and will be`);
|
||||
console.log(` cancelled after a ${Math.round(ttlNoticeGraceMs() / 60_000)}min grace window`);
|
||||
console.log(` (auditable error_text; visible in 'gbrain jobs stats').`);
|
||||
}
|
||||
console.log(` Tune or disable: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`);
|
||||
console.log('');
|
||||
await engine.setConfig(TTL_NOTICE_SHOWN_KEY, new Date().toISOString());
|
||||
}
|
||||
} catch {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// #3390: ZeroEntropy sunset notice. ZE announced (2026-07-24) that
|
||||
// its hosted endpoints — including /models/embed and /models/rerank —
|
||||
// shut down on 2026-09-04. Any brain resolving to a zeroentropyai:*
|
||||
|
||||
+237
-204
@@ -1,239 +1,272 @@
|
||||
/**
|
||||
* v0.36.0.0 — `gbrain ze-switch` CLI lever for the ZeroEntropy default switch.
|
||||
* `gbrain ze-switch` — RETIRED refusal/redirect shim.
|
||||
*
|
||||
* Subcommands / flags:
|
||||
* gbrain ze-switch Run the interactive prompt
|
||||
* gbrain ze-switch --dry-run Plan only; change nothing
|
||||
* gbrain ze-switch --json Machine-readable envelope
|
||||
* gbrain ze-switch --non-interactive Switch without prompting
|
||||
* (errors if ZEROENTROPY_API_KEY missing
|
||||
* unless --ignore-missing-key is also set)
|
||||
* gbrain ze-switch --resume Finish a half-applied switch (recovery)
|
||||
* gbrain ze-switch --force Bypass the `prompt_shown` gate
|
||||
* (use after `n` / never-ask-again)
|
||||
* gbrain ze-switch --undo Reverse: restore prior model + dim
|
||||
* + reranker state. Cost-warning prompt
|
||||
* appears before any change.
|
||||
* gbrain ze-switch --undo --non-interactive --confirm-reembed
|
||||
* Scripted undo path (also pays for re-embed)
|
||||
* ZeroEntropy's hosted API shuts down on ZEROENTROPY_SUNSET_DATE. Every
|
||||
* invocation refuses or redirects; nothing here mutates the brain:
|
||||
*
|
||||
* gbrain ze-switch --help Truthful usage (exit 0, engine-free)
|
||||
* gbrain ze-switch --undo [--json] Print the exact migration command that
|
||||
* returns this brain to its pre-switch
|
||||
* provider (from the stored snapshot).
|
||||
* Guidance only — exit 1, nothing changes.
|
||||
* anything else Refusal naming the canonical migration.
|
||||
*
|
||||
* Why the legacy actions are gone: the forward switch/resume have been
|
||||
* sunset-refused since v0.46.3, and the undo ACTION wrote DB-plane config
|
||||
* (engine.setConfig) that the post-v0.37 file-plane-canonical embed pipeline
|
||||
* never reads — it could rebuild the schema (dropping every vector) while the
|
||||
* runtime kept resolving the old model. Printing the verified, resumable
|
||||
* `gbrain migrate embeddings` command is strictly safer than acting.
|
||||
*
|
||||
* The whole command is deleted in the v0.47 September removal release.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import {
|
||||
planRetrievalUpgrade,
|
||||
applyRetrievalUpgrade,
|
||||
resumeRetrievalUpgrade,
|
||||
undoRetrievalUpgrade,
|
||||
formatEnvOverrideWarning,
|
||||
type ApplyResult,
|
||||
} from '../core/retrieval-upgrade-planner.ts';
|
||||
import {
|
||||
runRetrievalUpgradePrompt,
|
||||
runUndoPrompt,
|
||||
} from '../core/retrieval-upgrade-prompt.ts';
|
||||
ZEROENTROPY_SUNSET_DATE,
|
||||
renderCanonicalMigrationCommands,
|
||||
} from '../core/ai/defaults.ts';
|
||||
import { getCliOptions } from '../core/cli-options.ts';
|
||||
|
||||
interface Flags {
|
||||
dryRun: boolean;
|
||||
json: boolean;
|
||||
nonInteractive: boolean;
|
||||
resume: boolean;
|
||||
force: boolean;
|
||||
undo: boolean;
|
||||
confirmReembed: boolean;
|
||||
ignoreMissingKey: boolean;
|
||||
ignoreEnvOverride: boolean;
|
||||
/** Config row written by the pre-v0.46.3 forward switch (the literal matches
|
||||
* KEY_PREVIOUS_SNAPSHOT in retrieval-upgrade-planner.ts; kept local so the
|
||||
* shim does not drag the retired planner module into its import graph). */
|
||||
const KEY_PREVIOUS_SNAPSHOT = 'ze_switch_previous_snapshot';
|
||||
|
||||
interface ZeSwitchSnapshot {
|
||||
embedding_model: string;
|
||||
embedding_dimensions: number;
|
||||
search_reranker_enabled?: boolean;
|
||||
search_reranker_model?: string | null;
|
||||
}
|
||||
|
||||
function parseFlags(args: string[]): Flags {
|
||||
return {
|
||||
dryRun: args.includes('--dry-run'),
|
||||
json: args.includes('--json'),
|
||||
nonInteractive: args.includes('--non-interactive') || args.includes('--yes'),
|
||||
resume: args.includes('--resume'),
|
||||
force: args.includes('--force'),
|
||||
undo: args.includes('--undo'),
|
||||
confirmReembed: args.includes('--confirm-reembed'),
|
||||
ignoreMissingKey: args.includes('--ignore-missing-key'),
|
||||
// v0.41.2.1: escape hatch for power users running parallel experiments
|
||||
// with GBRAIN_EMBEDDING_MODEL set. Loud stderr line when used.
|
||||
ignoreEnvOverride: args.includes('--ignore-env-override'),
|
||||
};
|
||||
// Retired forward-switch flags — kept as quoted literals ONLY so the
|
||||
// generated CLI_FLAG_REGISTRY row keeps accepting them and old scripts reach
|
||||
// the refusal message naming the migration instead of dying pre-dispatch
|
||||
// with an unknown-flag error (cli.ts validates against the row BEFORE
|
||||
// dispatch; the row is generated from these literals, and safety flags like
|
||||
// '--dry-run' need quoted consumption evidence to survive regeneration).
|
||||
// The shim never consults them — every non-help/undo invocation refuses.
|
||||
// '--markdown' rode the pre-shim row (generator over-scan); kept for the
|
||||
// same old-scripts-reach-the-refusal reason. The registry-superset pin in
|
||||
// test/ze-switch-cli.test.ts makes any drop of this list loud.
|
||||
export const RETIRED_FLAGS = [
|
||||
'--dry-run',
|
||||
'--resume',
|
||||
'--force',
|
||||
'--non-interactive',
|
||||
'--yes',
|
||||
'--ignore-missing-key',
|
||||
'--ignore-env-override',
|
||||
'--confirm-reembed',
|
||||
'--markdown',
|
||||
];
|
||||
|
||||
/** The `--brain <id>` selector is parsed and STRIPPED by the global CLI
|
||||
* option layer before dispatch, so any command this shim tells the user to
|
||||
* run must carry it explicitly — otherwise `ze-switch --brain team-x --undo`
|
||||
* reads team-x's snapshot but the printed migrate command targets the
|
||||
* ambient/default brain (a paid re-embed of the wrong corpus). */
|
||||
function brainSuffix(): string {
|
||||
const brain = getCliOptions().brain;
|
||||
return brain ? ` --brain ${brain}` : '';
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write(`Usage: gbrain ze-switch [flags]
|
||||
const cmds = renderCanonicalMigrationCommands();
|
||||
process.stdout.write(`Usage: gbrain ze-switch [--undo] [--json]
|
||||
|
||||
Switch the brain's embedding + reranker defaults to ZeroEntropy.
|
||||
RETIRED — ZeroEntropy shuts down its hosted API on ${ZEROENTROPY_SUNSET_DATE}.
|
||||
Switching a brain ONTO ZeroEntropy is refused (exit 1, reason
|
||||
provider_sunset), and the legacy dry-run/resume/undo ACTIONS no longer run.
|
||||
Every invocation refuses or redirects; nothing changes your brain.
|
||||
|
||||
Flags:
|
||||
--dry-run Plan only; change nothing.
|
||||
--json Machine-readable output.
|
||||
--non-interactive Skip prompts; apply directly (CI / scripts).
|
||||
--resume Finish a half-applied switch (crash recovery).
|
||||
--force Bypass the prompt_shown gate (use after --undo or "never ask").
|
||||
--undo Reverse the switch: restore prior model + dim + reranker.
|
||||
--confirm-reembed Required with --undo --non-interactive (re-embed pays cost).
|
||||
--ignore-missing-key Allow --non-interactive without ZEROENTROPY_API_KEY set.
|
||||
--ignore-env-override Apply even when GBRAIN_EMBEDDING_* env vars would
|
||||
override the target at runtime (use if you know why).
|
||||
--help Show this help.
|
||||
--undo Print the exact migration command that returns this brain to its
|
||||
pre-switch provider (read from the stored switch snapshot). No
|
||||
changes are made; run the printed command yourself. Exit 1.
|
||||
--json Machine-readable envelope on stdout.
|
||||
--help This help. Exit 0.
|
||||
|
||||
To LEAVE ZeroEntropy (the maintained path):
|
||||
${cmds.recommendedDryRun} # cost preview
|
||||
${cmds.recommended}
|
||||
Playbook: skills/migrations/v0.46.3.0.md
|
||||
|
||||
Retired flags — still accepted so old scripts get the refusal above instead
|
||||
of an unknown-flag error: ${RETIRED_FLAGS.join(' ')}
|
||||
|
||||
This command is deleted in the September (v0.47) removal release.
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render an ApplyResult; if status is 'refused' (env-override gate),
|
||||
* write the ASCII warning box to stderr AND exit non-zero. Pure data
|
||||
* stays in the JSON envelope; the box is for human readers.
|
||||
*/
|
||||
function renderApplyResult(result: ApplyResult, json: boolean): void {
|
||||
if (result.status === 'refused' && result.reason === 'env_override') {
|
||||
if (json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.error(formatEnvOverrideWarning(result.warning));
|
||||
console.error(`\nSwitch status: refused (env_override)`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
if (json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`Switch status: ${result.status}`);
|
||||
}
|
||||
function refusalEnvelope(
|
||||
extraMessage?: string,
|
||||
opts: { omitUndoHint?: boolean } = {},
|
||||
): {
|
||||
status: 'refused';
|
||||
reason: 'provider_sunset';
|
||||
/** The LIVE canonical migration command — what an agent should run. */
|
||||
migrate: string;
|
||||
/** The cost-preview variant — run this first. */
|
||||
migrate_preview: string;
|
||||
message: string;
|
||||
} {
|
||||
const cmds = renderCanonicalMigrationCommands();
|
||||
const brain = brainSuffix();
|
||||
const live = `${cmds.recommended}${brain}`;
|
||||
const preview = `${cmds.recommendedDryRun}${brain}`;
|
||||
const message =
|
||||
(extraMessage ? `${extraMessage}\n` : '') +
|
||||
`ze-switch is retired: ZeroEntropy shuts down its hosted API on ${ZEROENTROPY_SUNSET_DATE}.\n` +
|
||||
`To LEAVE ZeroEntropy: ${preview} # cost preview\n` +
|
||||
` then: ${live}\n` +
|
||||
`Playbook: skills/migrations/v0.46.3.0.md` +
|
||||
// Never point a failed --undo back at --undo (guidance loop).
|
||||
(opts.omitUndoHint
|
||||
? ''
|
||||
: `\nTo see the command that returns this brain to its pre-switch provider: gbrain ze-switch --undo`);
|
||||
return { status: 'refused', reason: 'provider_sunset', migrate: live, migrate_preview: preview, message };
|
||||
}
|
||||
|
||||
export async function runZeSwitch(args: string[], engine: BrainEngine): Promise<void> {
|
||||
/** Emit the envelope (stdout JSON or stderr message) and exit 1. */
|
||||
function emitAndExit(payload: { message: string } & Record<string, unknown>, json: boolean): never {
|
||||
if (json) {
|
||||
console.log(JSON.stringify(payload));
|
||||
} else {
|
||||
console.error(payload.message);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/** Model ids are `provider:model` tokens; the tail may nest (`ollama:model:tag`,
|
||||
* `openrouter:google/gemma`, `nvidia:nvidia/nv-embedqa-e5-v5`). The snapshot
|
||||
* row is data-plane content (writable via config set / direct DB / a mounted
|
||||
* brain), and its fields land verbatim in a command the user or a downstream
|
||||
* agent is told to RUN — so validate before interpolating and degrade to the
|
||||
* plain refusal on anything suspicious. Leading alphanumeric + no whitespace
|
||||
* means a value like `--force-sunset-target` can never inject a flag. */
|
||||
const MODEL_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*:[A-Za-z0-9._/:-]+$/;
|
||||
|
||||
type SnapshotReadResult =
|
||||
| { kind: 'ok'; snapshot: ZeSwitchSnapshot }
|
||||
| { kind: 'missing' }
|
||||
| { kind: 'invalid' }
|
||||
| { kind: 'read_error' };
|
||||
|
||||
function parseSnapshot(raw: string): ZeSwitchSnapshot | null {
|
||||
try {
|
||||
const p = JSON.parse(raw) as ZeSwitchSnapshot;
|
||||
if (
|
||||
p &&
|
||||
typeof p.embedding_model === 'string' &&
|
||||
MODEL_ID_RE.test(p.embedding_model) &&
|
||||
Number.isInteger(p.embedding_dimensions) &&
|
||||
p.embedding_dimensions > 0 &&
|
||||
// A string "false" would pass a truthiness check and then FAIL the
|
||||
// strict ===false test below, re-enabling a reranker the snapshot says
|
||||
// was off — require boolean or absent.
|
||||
(p.search_reranker_enabled == null || typeof p.search_reranker_enabled === 'boolean') &&
|
||||
(p.search_reranker_model == null ||
|
||||
(typeof p.search_reranker_model === 'string' && MODEL_ID_RE.test(p.search_reranker_model)))
|
||||
) {
|
||||
return p;
|
||||
}
|
||||
} catch {
|
||||
/* corrupt JSON is `invalid` — the caller words the refusal */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Pure builder for the undo redirect commands (exported for tests). */
|
||||
export function buildUndoCommands(
|
||||
snapshot: ZeSwitchSnapshot,
|
||||
brainArg: string,
|
||||
): { live: string; preview: string } {
|
||||
// Fold the pre-switch reranker into the same run: `--reranker` takes a
|
||||
// model id or `off`; omitted means the migration's own default.
|
||||
// enabled===false WINS over a lingering model id — the pre-switch brain
|
||||
// had reranking off, and `migrate embeddings --reranker <model>` would
|
||||
// re-enable it (the retired undo restored `enabled` independently).
|
||||
const rerankerArg =
|
||||
snapshot.search_reranker_enabled === false
|
||||
? ' --reranker off'
|
||||
: snapshot.search_reranker_model
|
||||
? ` --reranker ${snapshot.search_reranker_model}`
|
||||
: '';
|
||||
const live = `gbrain migrate embeddings --to ${snapshot.embedding_model} --dim ${snapshot.embedding_dimensions}${rerankerArg}${brainArg}`;
|
||||
return { live, preview: `${live} --dry-run` };
|
||||
}
|
||||
|
||||
/** cli.ts SELF_HELP_WITHOUT_ENGINE adapter: that record's handlers take
|
||||
* (engine, args); runZeSwitch takes (args, engine). Help never touches the
|
||||
* engine, so null is safe here. */
|
||||
export function runZeSwitchSelfHelp(_engine: never, args: string[]): Promise<void> {
|
||||
return runZeSwitch(args, null);
|
||||
}
|
||||
|
||||
export async function runZeSwitch(args: string[], engine: BrainEngine | null): Promise<void> {
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const flags = parseFlags(args);
|
||||
// Both --json spellings, mirroring cli.ts's own convention.
|
||||
const json = args.some((a) => a === '--json' || (a.startsWith('--json=') && a !== '--json=false'));
|
||||
|
||||
// v0.46.3: ZeroEntropy is shutting down. Switching a brain ONTO it — including
|
||||
// resuming a half-applied forward switch — is disabled; only --undo (which
|
||||
// moves a brain OFF it) and --dry-run (read-only plan) still run. The whole
|
||||
// command is deleted in the September removal release.
|
||||
if (!flags.undo && !flags.dryRun) {
|
||||
const {
|
||||
ZEROENTROPY_SUNSET_DATE,
|
||||
renderCanonicalMigrationCommands,
|
||||
} = await import('../core/ai/defaults.ts');
|
||||
const msg =
|
||||
`ze-switch is disabled: ZeroEntropy shuts down its hosted API on ${ZEROENTROPY_SUNSET_DATE}.\n` +
|
||||
'Switching onto it (or resuming a half-applied switch) would strand this brain.\n' +
|
||||
`To LEAVE ZeroEntropy: ${renderCanonicalMigrationCommands().recommendedDryRun}\n` +
|
||||
'To undo a prior switch: gbrain ze-switch --undo';
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify({ status: 'refused', reason: 'provider_sunset', message: msg }));
|
||||
} else {
|
||||
console.error(msg);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
// --dry-run: just plan, never apply.
|
||||
if (flags.dryRun) {
|
||||
const plan = await planRetrievalUpgrade(engine);
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify({ status: 'planned', plan }, null, 2));
|
||||
} else {
|
||||
console.log(`Current model: ${plan.current_embedding_model} (${plan.current_dim}d)`);
|
||||
console.log(`Target model: ${plan.target_embedding_model ?? '(no change)'}`);
|
||||
console.log(`Target dim: ${plan.target_dim ?? '(no change)'}`);
|
||||
console.log(`Pages pending: chunker=${plan.pages_pending_chunker}, dim=${plan.pages_pending_dim}`);
|
||||
console.log(`Est cost: $${plan.est_cost_usd.toFixed(2)}`);
|
||||
console.log(`Est minutes: ${plan.est_minutes}`);
|
||||
console.log(`Schema change: ~${plan.est_schema_change_seconds}s`);
|
||||
console.log(`Offered: ${plan.ze_switch_offered}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// --resume: complete a half-applied switch.
|
||||
if (flags.resume) {
|
||||
if (flags.ignoreEnvOverride) {
|
||||
console.error('[ze-switch] WARNING: --ignore-env-override is set; env vars will silently override the switch at runtime.');
|
||||
}
|
||||
const result = await resumeRetrievalUpgrade(engine, {
|
||||
ignoreEnvOverride: flags.ignoreEnvOverride,
|
||||
});
|
||||
// v0.41.2.1: route through the env-override-aware renderer so
|
||||
// refused-status emits the ASCII warning box + exits non-zero.
|
||||
if (result.status === 'refused' && result.reason === 'env_override') {
|
||||
renderApplyResult(result, flags.json); // exits non-zero
|
||||
}
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`Resume status: ${result.status}`);
|
||||
}
|
||||
process.exit(result.status === 'applied' || result.status === 'skipped_already_applied' ? 0 : 1);
|
||||
}
|
||||
|
||||
// --undo: reverse switch.
|
||||
if (flags.undo) {
|
||||
if (flags.nonInteractive) {
|
||||
if (!flags.confirmReembed) {
|
||||
console.error('--undo --non-interactive requires --confirm-reembed (undo re-embeds at the prior width — costs real money).');
|
||||
process.exit(1);
|
||||
if (args.includes('--undo')) {
|
||||
// Read the pre-switch snapshot the old forward path stored. A missing,
|
||||
// corrupt, invalid-shape, or unreadable snapshot degrades to the plain
|
||||
// refusal (there is nothing to redirect to); `redirected` is reserved
|
||||
// for a validated snapshot. The failure states word the refusal
|
||||
// differently — telling the operator of a switched brain whose snapshot
|
||||
// failed validation that "no switch was recorded" would be false, and a
|
||||
// null engine here means the brain could not be reached at all.
|
||||
let read: SnapshotReadResult = engine ? { kind: 'missing' } : { kind: 'read_error' };
|
||||
if (engine) {
|
||||
try {
|
||||
const raw = await engine.getConfig(KEY_PREVIOUS_SNAPSHOT);
|
||||
if (raw) {
|
||||
const parsed = parseSnapshot(raw);
|
||||
read = parsed ? { kind: 'ok', snapshot: parsed } : { kind: 'invalid' };
|
||||
}
|
||||
const result = await undoRetrievalUpgrade(engine);
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`Undo status: ${result.status}`);
|
||||
}
|
||||
process.exit(result.status === 'undone' ? 0 : 1);
|
||||
} catch {
|
||||
read = { kind: 'read_error' };
|
||||
}
|
||||
// Interactive undo: shows cost-warning prompt.
|
||||
const result = await runUndoPrompt(engine);
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === 'undone' ? 0 : 1);
|
||||
}
|
||||
|
||||
// --non-interactive: apply without prompting.
|
||||
if (flags.nonInteractive) {
|
||||
if (!process.env.ZEROENTROPY_API_KEY && !flags.ignoreMissingKey) {
|
||||
const config = await engine.getConfig('zeroentropy_api_key');
|
||||
if (!config) {
|
||||
console.error('ZEROENTROPY_API_KEY not set. Pass --ignore-missing-key to switch anyway (embeddings will fail until you set a key).');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
if (flags.ignoreEnvOverride) {
|
||||
console.error('[ze-switch] WARNING: --ignore-env-override is set; env vars will silently override the switch at runtime.');
|
||||
}
|
||||
const plan = await planRetrievalUpgrade(engine);
|
||||
const result = await applyRetrievalUpgrade(engine, plan, {
|
||||
ignoreEnvOverride: flags.ignoreEnvOverride,
|
||||
});
|
||||
// v0.41.2.1: render env-override refusal with ASCII box + exit non-zero.
|
||||
if (result.status === 'refused' && result.reason === 'env_override') {
|
||||
renderApplyResult(result, flags.json); // exits non-zero
|
||||
}
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`Switch status: ${result.status}`);
|
||||
}
|
||||
process.exit(
|
||||
result.status === 'applied' || result.status === 'skipped_already_applied' || result.status === 'skipped_no_work'
|
||||
? 0
|
||||
: 1,
|
||||
if (read.kind === 'ok') {
|
||||
const { live, preview } = buildUndoCommands(read.snapshot, brainSuffix());
|
||||
const message =
|
||||
`ze-switch no longer undoes in place (the retired action wrote config the runtime does not read).\n` +
|
||||
`To return this brain to its pre-switch provider, run:\n` +
|
||||
` ${preview} # cost preview\n` +
|
||||
` ${live}\n` +
|
||||
`(This reflects the recorded pre-switch snapshot; the preview shows the live\n` +
|
||||
` current->target plan and the migration verifies against the database before\n` +
|
||||
` changing anything — a brain that already migrated will report nothing to do.)`;
|
||||
emitAndExit(
|
||||
{
|
||||
status: 'redirected',
|
||||
reason: 'provider_sunset',
|
||||
undo_command: live,
|
||||
undo_preview: preview,
|
||||
message,
|
||||
},
|
||||
json,
|
||||
);
|
||||
}
|
||||
|
||||
// Interactive mode.
|
||||
const result = await runRetrievalUpgradePrompt(engine, { force: flags.force });
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === 'applied' || result.status === 'declined_this_run' || result.status === 'declined_forever' || result.status === 'non_tty_skip' || result.status === 'not_offered' ? 0 : 1);
|
||||
} finally {
|
||||
// Engine lifecycle is owned by the dispatcher.
|
||||
const undoFailure =
|
||||
read.kind === 'invalid'
|
||||
? 'A switch snapshot exists but is unreadable or failed validation — inspect the ze_switch_previous_snapshot config row before trusting any undo guidance.'
|
||||
: read.kind === 'read_error'
|
||||
? 'Could not read the switch snapshot (no brain configured, or the config read failed) — check the brain connection and retry.'
|
||||
: 'No prior switch snapshot recorded — nothing to undo.';
|
||||
emitAndExit(refusalEnvelope(undoFailure, { omitUndoHint: true }), json);
|
||||
}
|
||||
|
||||
// Every other invocation — bare, --dry-run, --resume, --non-interactive,
|
||||
// --force, any combination — refuses.
|
||||
emitAndExit(refusalEnvelope(), json);
|
||||
}
|
||||
|
||||
@@ -30,7 +30,10 @@ export const collectStalledJobs: AdvisorCollector = {
|
||||
severity: 'warn',
|
||||
title: `${r.n} "${r.name}" job${r.n === 1 ? '' : 's'} look stalled (lock lapsed / retrying).`,
|
||||
detail: 'A wedged worker stops backfill/sync from progressing.',
|
||||
fix: { command_argv: ['gbrain', 'jobs', 'status'] },
|
||||
// 'jobs stats' is the real subcommand — 'jobs status' never existed
|
||||
// (the dead fix-command shipped unnoticed because nothing executes
|
||||
// advisor fixes automatically).
|
||||
fix: { command_argv: ['gbrain', 'jobs', 'stats'] },
|
||||
collector: 'stalled-jobs',
|
||||
ask_user: true,
|
||||
});
|
||||
|
||||
+2
-2
@@ -72,8 +72,8 @@ export function isValidZeroEntropyDim(dims: number): boolean {
|
||||
// Matryoshka — any positive integer up to the model's native size. When a
|
||||
// brain is configured with `embedding_dimensions` OUTSIDE that range, OpenAI
|
||||
// returns HTTP 400 at first embed. We catch it locally with a paste-ready
|
||||
// fix so users don't see opaque "vector dimension mismatch" errors after
|
||||
// `gbrain ze-switch --undo` lands them on OpenAI at the wrong dim.
|
||||
// fix so users don't see opaque "vector dimension mismatch" errors after a
|
||||
// `gbrain migrate embeddings --to openai:...` lands them at the wrong dim.
|
||||
const OPENAI_TEXT3_MAX_DIMS: Record<string, number> = {
|
||||
'text-embedding-3-small': 1536,
|
||||
'text-embedding-3-large': 3072,
|
||||
|
||||
@@ -2730,7 +2730,8 @@ export interface ChatToolDef {
|
||||
*/
|
||||
/**
|
||||
* Default per-call max output tokens. Thinking-by-default Claude 5 models
|
||||
* (`anthropic:claude-*-5`) burn a large chunk of the budget on internal
|
||||
* (`anthropic:claude-*-5`, including routed forms like
|
||||
* `openrouter:anthropic/claude-*-5`) burn a large chunk of the budget on internal
|
||||
* reasoning before emitting any text, so a 4096 default leaves them with empty
|
||||
* final text on the subagent tool loop. Give those models headroom; providers
|
||||
* bill actual tokens, not the cap, so it is free for the models that don't use
|
||||
@@ -2741,7 +2742,7 @@ export interface ChatToolDef {
|
||||
*/
|
||||
const DEFAULT_MAX_OUTPUT_TOKENS = 4096;
|
||||
const THINKING_MODEL_MAX_OUTPUT_TOKENS = 32000;
|
||||
const THINKING_BY_DEFAULT_MODEL_RE = /^anthropic[:/]claude-[a-z0-9]+-5(?:[.-]|$)/i;
|
||||
const THINKING_BY_DEFAULT_MODEL_RE = /(?:^|[:/])anthropic[:/]claude-[a-z0-9]+-5(?:[.-]|$)/i;
|
||||
function defaultMaxOutputTokens(modelStr: string | undefined): number {
|
||||
return modelStr && THINKING_BY_DEFAULT_MODEL_RE.test(modelStr)
|
||||
? THINKING_MODEL_MAX_OUTPUT_TOKENS
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Atomic file write for brain-repo markdown writers.
|
||||
*
|
||||
* Write path: unique tmp sibling → write → fsync → close → (optional verify
|
||||
* of the on-disk bytes) → chmod to the original mode → rename over the target.
|
||||
* The rename is atomic on POSIX filesystems, so readers never observe a torn
|
||||
* file; a crash mid-write leaves only a tmp sibling, never a corrupt target.
|
||||
*
|
||||
* The tmp name embeds pid + random bytes so concurrent writers (two fixers,
|
||||
* a fixer racing a render) can never collide on the tmp path itself. Note the
|
||||
* rename does NOT prevent lost updates between two read-modify-write writers —
|
||||
* callers that need that take the per-page lock (src/core/page-lock.ts).
|
||||
*
|
||||
* Every module used to roll its own copy of this pattern (write-through,
|
||||
* skillopt, schema-pack/mutate, self-upgrade, …). This is the shared home;
|
||||
* migrating the older copies is tracked in TODOS.md.
|
||||
*/
|
||||
|
||||
import {
|
||||
chmodSync,
|
||||
closeSync,
|
||||
existsSync,
|
||||
fsyncSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
statSync,
|
||||
unlinkSync,
|
||||
writeSync,
|
||||
} from 'fs';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { dirname } from 'path';
|
||||
|
||||
export interface AtomicWriteOpts {
|
||||
/**
|
||||
* Called with the bytes read back from the tmp file BEFORE the rename.
|
||||
* Throw to abort the write — the tmp file is removed and the target is
|
||||
* left untouched. Use this to validate that what actually landed on disk
|
||||
* still parses (backlinks uses parseMarkdown here).
|
||||
*/
|
||||
verify?: (onDisk: string) => void;
|
||||
}
|
||||
|
||||
export function atomicWriteFileSync(filePath: string, content: string, opts?: AtomicWriteOpts): void {
|
||||
const tmpPath = `${filePath}.tmp.${process.pid}.${randomBytes(4).toString('hex')}`;
|
||||
|
||||
// Preserve the target's mode across the rename (a fresh tmp file gets the
|
||||
// process umask, which can silently drop e.g. group-write bits).
|
||||
let mode: number | null = null;
|
||||
try {
|
||||
if (existsSync(filePath)) mode = statSync(filePath).mode & 0o7777;
|
||||
} catch {
|
||||
/* stat raced a delete — fall through with default mode */
|
||||
}
|
||||
|
||||
try {
|
||||
const fd = openSync(tmpPath, 'w', mode ?? 0o644);
|
||||
try {
|
||||
// Loop until every byte lands: writeSync may legally return a short
|
||||
// count under disk pressure/quotas, and a silent short write that
|
||||
// truncates AFTER valid frontmatter would pass a frontmatter-only
|
||||
// verifier and atomically install truncated content.
|
||||
const buf = Buffer.from(content, 'utf-8');
|
||||
let off = 0;
|
||||
while (off < buf.length) {
|
||||
const n = writeSync(fd, buf, off, buf.length - off);
|
||||
if (n <= 0) throw new Error(`atomic-write: short write at offset ${off}/${buf.length}`);
|
||||
off += n;
|
||||
}
|
||||
fsyncSync(fd);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
// open(2)'s mode argument is masked by the process umask (0664 & ~022 →
|
||||
// 0644), so an explicit chmod is required to actually PRESERVE the
|
||||
// target's mode across the rename — the pre-wave in-place write kept the
|
||||
// inode's mode exactly; this keeps that property.
|
||||
if (mode !== null) chmodSync(tmpPath, mode);
|
||||
if (opts?.verify) {
|
||||
opts.verify(readFileSync(tmpPath, 'utf-8'));
|
||||
}
|
||||
renameSync(tmpPath, filePath);
|
||||
// Durability of the RENAME itself: fsync the parent directory so a power
|
||||
// loss can't silently drop the new directory entry (the target is never
|
||||
// corrupt either way — this closes the write-vanished window). Dir fsync
|
||||
// is unsupported on some platforms; best-effort by design.
|
||||
try {
|
||||
const dfd = openSync(dirname(filePath), 'r');
|
||||
try { fsyncSync(dfd); } finally { closeSync(dfd); }
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
} catch (err) {
|
||||
try {
|
||||
if (existsSync(tmpPath)) unlinkSync(tmpPath);
|
||||
} catch {
|
||||
/* best-effort cleanup */
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* Capture content helpers — moved from src/commands/capture.ts in the
|
||||
* CLI→MCP gap-closure wave so the `capture` MCP op and the CLI share slug
|
||||
* defaulting, dedupe hashing, the binary guard, and the frontmatter merge.
|
||||
* (capture.ts statically imports operations.ts, so operations-layer code must
|
||||
* never import capture.ts — this module breaks that cycle.) Pure functions;
|
||||
* no fs, no engine.
|
||||
*/
|
||||
|
||||
import matter from 'gray-matter';
|
||||
import { computeContentHash } from './ingestion/types.ts';
|
||||
|
||||
/** The subset of capture options the frontmatter/slug helpers consume. */
|
||||
export interface CaptureFrontmatterOpts {
|
||||
type?: string;
|
||||
/** Ingestion channel override (CLI --source; ops never set it). */
|
||||
source?: string;
|
||||
/**
|
||||
* Channel label used as the `captured_via` DEFAULT when no user frontmatter
|
||||
* or CLI `--source` override is present. Lets a remote MCP caller record
|
||||
* `capture-mcp` provenance instead of the local-CLI-implying `capture-cli`.
|
||||
* Ordered below `source` so the explicit CLI flag still wins.
|
||||
*/
|
||||
capturedVia?: string;
|
||||
// v0.42.x — Life Chronicle (#2390) `--type event` sugar.
|
||||
who?: string;
|
||||
what?: string;
|
||||
where?: string;
|
||||
kind?: string;
|
||||
depth?: string;
|
||||
}
|
||||
|
||||
// v0.42.x — Life Chronicle (#2390): route the default slug prefix by type so
|
||||
// `gbrain capture --type diary` lands under life/diary/ and `--type event`
|
||||
// under life/events/ (matching the chronicle path-prefix inference). Everything
|
||||
// else keeps the inbox/ default.
|
||||
export function slugPrefixForType(type?: string): string {
|
||||
if (type === 'diary') return 'life/diary';
|
||||
if (type === 'event') return 'life/events';
|
||||
return 'inbox';
|
||||
}
|
||||
|
||||
export function defaultSlug(content: string, now: Date = new Date(), type?: string): string {
|
||||
const y = now.getUTCFullYear();
|
||||
const m = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
const d = String(now.getUTCDate()).padStart(2, '0');
|
||||
const hashPrefix = computeContentHash(content).slice(0, 8);
|
||||
return `${slugPrefixForType(type)}/${y}-${m}-${d}-${hashPrefix}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 CV10 — binary file guard. Scans the first 8KB of `buf` for a
|
||||
* NUL byte (0x00). Real text files (including UTF-8 with multi-byte CJK,
|
||||
* emoji, BOM) never contain a NUL byte at any position — text encoding
|
||||
* uses non-zero continuation bytes. NUL appears in binary formats:
|
||||
* executables, archives, compressed images, PDFs (after the magic-byte
|
||||
* header), most office documents. Single-pass scan; constant memory.
|
||||
*
|
||||
* Returns the 0-indexed byte offset of the first NUL, or -1 if clean.
|
||||
* Caller decides the error shape (message vs JSON envelope).
|
||||
*
|
||||
* Known limit: a PNG-without-NUL-in-first-8KB slips through. v0.39
|
||||
* magic-byte allowlist (per CV10-B + TODOS.md) closes this hole. The
|
||||
* 8KB ceiling bounds the scan cost to ~microseconds even on huge files.
|
||||
*/
|
||||
export function detectBinaryNullByte(buf: Buffer): number {
|
||||
const limit = Math.min(buf.length, 8 * 1024);
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (buf[i] === 0) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 CV9 — normalize content for content_hash so identical text
|
||||
* produces identical hashes regardless of leading/trailing whitespace,
|
||||
* line-ending style (CRLF vs LF), or Unicode normalization form. The
|
||||
* STORED body is preserved as-is (CRLF stays CRLF, BOM stays BOM).
|
||||
*
|
||||
* Two concerns, two transforms — the hash gets aggressive normalization
|
||||
* for dedup correctness; the stored body keeps user bytes for round-trip
|
||||
* fidelity. CQ2's CRLF/BOM preservation tests rely on this split.
|
||||
*/
|
||||
export function normalizeForHash(s: string): string {
|
||||
// Strip BOM, normalize line endings to LF, trim, NFKC for Unicode-stable hash.
|
||||
return s.replace(/^/, '').replace(/\r\n/g, '\n').trim().normalize('NFKC');
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a title from the first non-empty, non-`---` line of the body,
|
||||
* stripping leading markdown heading marks, capped at 80 chars. Truncation
|
||||
* is codepoint-aware (never splits an astral surrogate pair) and appends an
|
||||
* ellipsis so a cut title is visibly cut.
|
||||
* Falls back to 'Capture' when no usable line exists.
|
||||
*/
|
||||
export function deriveTitle(rawBody: string): string {
|
||||
const firstLine = rawBody
|
||||
.split('\n')
|
||||
.find((l) => l.trim().length > 0 && l.trim() !== '---') ?? '';
|
||||
const stripped = firstLine.replace(/^#+\s*/, '');
|
||||
const cps = [...stripped];
|
||||
return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture';
|
||||
}
|
||||
|
||||
// v0.42.x — Life Chronicle (#2390): assemble the `event:` frontmatter block
|
||||
// from the --who/--what/--where/--kind/--depth flags (only for --type event).
|
||||
// Returns undefined when no event flags are set so non-event captures are
|
||||
// untouched.
|
||||
export function buildEventBlock(opts: CaptureFrontmatterOpts): Record<string, unknown> | undefined {
|
||||
if (opts.type !== 'event') return undefined;
|
||||
const who = opts.who ? opts.who.split(',').map((s) => s.trim()).filter(Boolean) : [];
|
||||
const block: Record<string, unknown> = {};
|
||||
if (opts.what) block.what = opts.what;
|
||||
if (who.length) block.who = who;
|
||||
if (opts.where) block.where = opts.where;
|
||||
if (opts.kind) block.kind = opts.kind;
|
||||
if (opts.depth) block.depth = opts.depth;
|
||||
return Object.keys(block).length ? block : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.39.3.0 (BUG-1): merge capture's auto-stamped fields with any existing
|
||||
* frontmatter in `rawBody`, rather than always prepending a second
|
||||
* frontmatter block. The pre-fix code stamped its own `---` block on top
|
||||
* of files that already had frontmatter, producing `title: '---'` (the
|
||||
* file's opening delimiter became the outer title) and two consecutive
|
||||
* frontmatter blocks the parser interpreted as the outer block + a body
|
||||
* starting with a horizontal rule.
|
||||
*
|
||||
* Precedence rules (user-wins by default):
|
||||
* - `type`: opts.type (CLI flag) > userFm.type > 'note'
|
||||
* - `title`: userFm.title > derived-from-body
|
||||
* - `captured_via`: userFm.captured_via > opts.source > opts.capturedVia > 'capture-cli'
|
||||
* (opts.capturedVia is the per-channel default — remote MCP
|
||||
* captures pass 'capture-mcp' so provenance isn't misreported
|
||||
* as local CLI; the explicit CLI --source override still wins)
|
||||
* - `captured_at`: userFm.captured_at > now (user can pre-stamp for retroactive
|
||||
* captures; see CQ2 test case 4)
|
||||
* - Any other user-declared keys (description, tags, slug, etc.) pass through verbatim.
|
||||
*
|
||||
* For files WITHOUT existing frontmatter, preserves the original behavior:
|
||||
* stamps a fresh frontmatter block, and if the body doesn't already look
|
||||
* like markdown (no `#` heading), wraps it under a `# {title}` heading.
|
||||
*/
|
||||
export function mergeCaptureFrontmatter(rawBody: string, opts: CaptureFrontmatterOpts): string {
|
||||
const nowIso = new Date().toISOString();
|
||||
// Detect frontmatter: leading `---\n` or `---\r\n`, tolerating leading BOM/whitespace.
|
||||
// We do NOT use the more permissive `startsWith('---')` because a body that opens
|
||||
// with a horizontal-rule like `--- separator ---` would false-positive.
|
||||
const trimmedStart = rawBody.replace(/^/, '');
|
||||
const hasFrontmatter = /^---\r?\n/.test(trimmedStart);
|
||||
|
||||
if (!hasFrontmatter) {
|
||||
// No existing frontmatter: stamp a fresh block and (if body lacks markdown
|
||||
// structure) wrap under a derived heading.
|
||||
const title = deriveTitle(rawBody);
|
||||
const fm: Record<string, unknown> = {
|
||||
type: opts.type ?? 'note',
|
||||
title,
|
||||
captured_via: opts.source ?? opts.capturedVia ?? 'capture-cli',
|
||||
captured_at: nowIso,
|
||||
};
|
||||
const ev = buildEventBlock(opts);
|
||||
if (ev) fm.event = ev;
|
||||
const looksMarkdown = /^#{1,6}\s/.test(rawBody.trimStart());
|
||||
const body = looksMarkdown ? rawBody : `# ${title}\n\n${rawBody}`;
|
||||
return matter.stringify(body, fm);
|
||||
}
|
||||
|
||||
// Existing frontmatter: parse, merge user-wins, re-emit as a SINGLE block.
|
||||
let parsed: matter.GrayMatterFile<string>;
|
||||
try {
|
||||
parsed = matter(rawBody);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`malformed frontmatter in capture input: ${e instanceof Error ? e.message : String(e)}`,
|
||||
);
|
||||
}
|
||||
const userFm = (parsed.data ?? {}) as Record<string, unknown>;
|
||||
const merged: Record<string, unknown> = {
|
||||
// Spread user's declared keys first so 'description', 'tags', etc. pass through.
|
||||
...userFm,
|
||||
// Then apply auto-fields with the precedence rules above. The explicit
|
||||
// assignment AFTER the spread is intentional: it lets us implement the
|
||||
// mixed precedence (CLI flag wins for `type`; user wins for `title`/
|
||||
// `captured_via`/`captured_at`) in one expression per key.
|
||||
type: opts.type ?? userFm.type ?? 'note',
|
||||
title: userFm.title ?? deriveTitle(parsed.content),
|
||||
captured_via: userFm.captured_via ?? opts.source ?? opts.capturedVia ?? 'capture-cli',
|
||||
captured_at: userFm.captured_at ?? nowIso,
|
||||
};
|
||||
// v0.42.x — merge the event block (user-declared keys win per-key).
|
||||
const ev = buildEventBlock(opts);
|
||||
if (ev || userFm.event) {
|
||||
merged.event = { ...(ev ?? {}), ...((userFm.event as Record<string, unknown>) ?? {}) };
|
||||
}
|
||||
return matter.stringify(parsed.content, merged);
|
||||
}
|
||||
@@ -22,12 +22,12 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'autopilot': ['--aliases', '--all', '--auto-fix', '--batch', '--brain', '--break-lock', '--by-type', '--check', '--dim', '--dimensions', '--dir', '--drain', '--embedding-dimensions', '--embedding-model', '--explain', '--fast', '--ff-only', '--fix', '--force', '--force-break-lock', '--force-retry', '--from-pages', '--help', '--http', '--include-null-signature', '--inject-bootstrap', '--inline', '--input', '--install', '--interval', '--json', '--markdown', '--max-age', '--max-rss', '--max-usd', '--migrate-only', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-inject', '--no-mutate', '--no-worker', '--non-interactive', '--now', '--once', '--output', '--path', '--pattern', '--pending', '--phase', '--refresh-cache', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--source', '--stale', '--status', '--supersessions', '--surface', '--swap-only', '--target', '--target-score', '--thin', '--timeout', '--to', '--token-ttl', '--uninstall', '--unsafe-bypass-dream-guard', '--user', '--version', '--yes'],
|
||||
'backfill': ['--aliases', '--all', '--batch-size', '--brain', '--concurrency', '--dry-run', '--fresh', '--help', '--include-null-signature', '--json', '--keep-index', '--list', '--max-errors', '--max-rows', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--resume', '--source', '--stale', '--supersessions', '--thin'],
|
||||
'bench': ['--baseline', '--brain', '--explain', '--force', '--from', '--help', '--json', '--label', '--lang', '--limit', '--markdown', '--multimodal', '--near-symbol', '--restore-only', '--source', '--stale', '--symbol-kind', '--thin', '--threshold-jaccard', '--threshold-latency-multiplier', '--threshold-top1', '--to', '--tool'],
|
||||
'book-mirror': ['--aliases', '--all', '--allow-empty', '--apply', '--asof', '--author', '--auto', '--background', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--brain-wide-max-cost-usd', '--budget-usd-per-day', '--by-mention', '--chapters-dir', '--content', '--context-file', '--date', '--days', '--dry-run', '--entities', '--explain', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--kind', '--limit', '--max-turns', '--max-usd', '--mode', '--model', '--multimodal', '--no-confirm', '--no-embedding', '--no-extract', '--no-follow', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--timeout-ms', '--title', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'book-mirror': ['--aliases', '--all', '--allow-empty', '--apply', '--asof', '--author', '--auto', '--background', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--brain-wide-max-cost-usd', '--budget-usd-per-day', '--by-mention', '--chapters-dir', '--content', '--context-file', '--date', '--days', '--dry-run', '--entities', '--explain', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--kind', '--limit', '--max-turns', '--max-usd', '--mode', '--model', '--multimodal', '--no-confirm', '--no-embedding', '--no-extract', '--no-follow', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--thin', '--timeout', '--timeout-ms', '--title', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'bootstrap': ['--abbrev-ref', '--abort', '--accept-visibility-change-consequences', '--active', '--all', '--allow-unverified-remote', '--auto', '--brain', '--branch', '--cached', '--compile', '--confirm', '--count', '--delete-brain', '--diff-filter', '--env', '--error-unmatch', '--exclude-standard', '--fast', '--file', '--flag', '--force', '--from-pages', '--full', '--gbrain-bin', '--get', '--git-dir', '--git-path', '--harness', '--heads', '--help', '--home', '--hostname', '--http', '--id', '--init', '--install', '--is-inside-work-tree', '--isolated', '--jq', '--json', '--local', '--mcp-even-if-plugin', '--minimal', '--name', '--name-only', '--no-capture', '--no-cron', '--no-embedding', '--no-hooks', '--no-verify', '--once', '--only', '--others', '--pat-file', '--path', '--pglite', '--porcelain', '--port', '--private', '--project', '--pure', '--push', '--push-only', '--quiet', '--rebase', '--remove', '--repair', '--scope', '--scopes', '--set', '--short', '--show', '--show-toplevel', '--skip', '--source', '--status', '--surface', '--token', '--token-name', '--token-ttl', '--unset-all', '--url', '--user-hooks', '--verify', '--version', '--visibility', '--workspace', '--yes'],
|
||||
'brainstorm': ['--aliases', '--all', '--brain', '--chunker-debug', '--code', '--compile', '--fast', '--file', '--fix', '--force', '--force-rechunk', '--force-resume', '--from-pages', '--full', '--help', '--http', '--include-null-signature', '--json', '--judge-model', '--lang', '--limit', '--list-runs', '--markdown', '--max-cost', '--max-far-set', '--max-ideas-per-judge-call', '--model', '--no-embed', '--no-embedding', '--no-extract', '--no-save', '--pattern', '--pending', '--reset', '--resolve', '--resume', '--retry-failed', '--retry-judge', '--save', '--source', '--stale', '--strict-budget', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--yes'],
|
||||
'cache': ['--brain', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--no-embedding', '--source', '--surface', '--token-ttl', '--yes'],
|
||||
'calibration': ['--ab', '--aliases', '--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--dry-run', '--entities', '--explain', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--holder', '--http', '--image', '--include-null-signature', '--json', '--key-prefix', '--kind', '--lang', '--limit', '--markdown', '--max-usd', '--mode', '--multimodal', '--near-symbol', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--phase', '--progress-interval', '--progress-json', '--quiet', '--regenerate', '--repo', '--reset', '--resolve', '--restore-only', '--save', '--scrub-gstack', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl', '--trusted-extraction', '--undo-wave', '--url', '--with-calibration', '--with-db', '--yes'],
|
||||
'call': ['--aliases', '--all', '--all-sources', '--as-context', '--auto-fix', '--background', '--brain', '--by-mention', '--catch-up', '--concurrency', '--confirm-destructive', '--content', '--cost-estimate', '--count', '--days', '--depth', '--dim', '--dir', '--direction', '--enable-dcr', '--enable-dcr-insecure', '--explain', '--fast', '--federated', '--file', '--fix', '--follow', '--force', '--from', '--from-meetings', '--grant-types', '--grep', '--hard-deadline', '--help', '--http', '--image', '--include-expired', '--include-frontmatter', '--include-null-signature', '--infer-dates', '--install', '--interval', '--json', '--key', '--kind', '--lang', '--limit', '--link-source', '--link-type', '--llm', '--migrate-only', '--missing-path', '--multimodal', '--ner', '--no-embed', '--no-expand', '--no-extract', '--no-federated', '--no-hard-deadline', '--no-retry-connect', '--no-save', '--older-than', '--page', '--param', '--params', '--password', '--path', '--pattern', '--pending', '--pglite', '--port', '--probe-pglite', '--progress-interval', '--progress-json', '--public-url', '--queue', '--quiet', '--reenrich-after', '--refresh-cache', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--restore-only', '--save', '--scopes', '--session', '--sigma', '--since', '--slug', '--slug-prefix', '--source', '--source-guard', '--source-id', '--stale', '--status', '--stdin', '--strategy', '--suites', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--synthesize', '--tag', '--thin', '--timeout', '--to', '--today', '--token', '--token-ttl', '--tools-json', '--type', '--uninstall', '--url', '--version', '--watch', '--with-calibration', '--workers', '--yes'],
|
||||
'calibration': ['--ab', '--aliases', '--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--dry-run', '--entities', '--explain', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--holder', '--http', '--image', '--include-null-signature', '--json', '--key-prefix', '--kind', '--lang', '--limit', '--markdown', '--max-usd', '--mode', '--multimodal', '--near-symbol', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--phase', '--progress-interval', '--progress-json', '--quiet', '--regenerate', '--repo', '--reset', '--resolve', '--restore-only', '--save', '--scrub-gstack', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl', '--trusted-extraction', '--undo-wave', '--url', '--with-calibration', '--with-db', '--yes'],
|
||||
'call': ['--aliases', '--all', '--all-sources', '--apply', '--as-context', '--auto-fix', '--background', '--brain', '--by-mention', '--catch-up', '--concurrency', '--confirm-destructive', '--content', '--cost-estimate', '--count', '--days', '--depth', '--dim', '--dir', '--direction', '--enable-dcr', '--enable-dcr-insecure', '--explain', '--fast', '--federated', '--file', '--fix', '--follow', '--force', '--from', '--from-meetings', '--grant-types', '--grep', '--hard-deadline', '--help', '--http', '--image', '--include-expired', '--include-frontmatter', '--include-null-signature', '--infer-dates', '--install', '--interval', '--json', '--key', '--kind', '--lang', '--limit', '--link-source', '--link-type', '--llm', '--migrate-only', '--missing-path', '--multimodal', '--ner', '--no-embed', '--no-expand', '--no-extract', '--no-federated', '--no-hard-deadline', '--no-retry-connect', '--no-save', '--older-than', '--page', '--param', '--params', '--password', '--path', '--pattern', '--pending', '--pglite', '--port', '--probe-pglite', '--progress-interval', '--progress-json', '--public-url', '--queue', '--quiet', '--reenrich-after', '--refresh-cache', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--restore-only', '--save', '--scopes', '--session', '--sigma', '--since', '--slug', '--slug-prefix', '--source', '--source-guard', '--source-id', '--stale', '--status', '--stdin', '--strategy', '--suites', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--synthesize', '--tag', '--thin', '--timeout', '--to', '--today', '--token', '--token-ttl', '--tools-json', '--type', '--undo', '--uninstall', '--url', '--version', '--watch', '--with-calibration', '--workers', '--yes'],
|
||||
'capture': ['--aliases', '--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--depth', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--repo', '--reset', '--resolve', '--restore-only', '--save', '--scopes', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--trusted-extraction', '--type', '--url', '--what', '--where', '--who', '--with-db', '--yes'],
|
||||
'check-backlinks': ['--background', '--brain', '--brain-wide-max-cost-usd', '--dir', '--dry-run', '--explain', '--follow', '--help', '--include-frontmatter', '--json', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--source', '--stale', '--timeout', '--type'],
|
||||
'check-resolvable': ['--brain', '--dry-run', '--fix', '--help', '--json', '--skills-dir', '--source', '--strict', '--verbose'],
|
||||
@@ -40,18 +40,18 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'config': ['--aliases', '--all', '--brain', '--column', '--coverage-override', '--detail', '--embedding-dimensions', '--embedding-model', '--fast', '--federated-read', '--follow', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--markdown', '--model', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--pattern', '--pending', '--pglite', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--surface', '--thin', '--token-ttl', '--yes'],
|
||||
'connect': ['--agent', '--auto', '--bearer-token-env-var', '--bind', '--brain', '--client-id', '--client-secret', '--delete-brain', '--env', '--force', '--grant-types', '--header', '--help', '--http', '--install', '--json', '--name', '--oauth', '--public-url', '--pure', '--register', '--remove', '--scope', '--scopes', '--show-token', '--source', '--status', '--timeout-ms', '--token', '--token-endpoint-auth-method', '--url', '--version', '--yes'],
|
||||
'conversation-parser': ['--aliases', '--all', '--brain', '--help', '--include-null-signature', '--json', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin'],
|
||||
'doctor': ['--ab', '--abbrev-ref', '--abi', '--abort', '--aliases', '--all', '--allow-shell-jobs', '--allow-unverified-remote', '--auto', '--auto-fix', '--auto-update', '--background', '--batch', '--brain', '--brain-wide-max-cost-usd', '--branch', '--break-lock', '--build-index', '--by-mention', '--by-type', '--cached', '--check', '--column', '--compile', '--concurrency', '--confidence', '--confirm', '--content-audit', '--count', '--days', '--delete-brain', '--detach', '--detail', '--diff-filter', '--dim', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--env', '--exclude-standard', '--exclusive', '--explain', '--fast', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--force-schema', '--force-sunset-target', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--gbrain-bin', '--get', '--git-dir', '--git-path', '--grant-types', '--harness', '--health-interval', '--help', '--history', '--home', '--http', '--id', '--ignore-env-override', '--ignore-missing-key', '--include-flagged', '--include-frontmatter', '--include-null-signature', '--include-pseudo', '--index-audit', '--init', '--input', '--install', '--is-inside-work-tree', '--job-isolation', '--jq', '--json', '--lang', '--limit', '--local', '--locks', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-crashes', '--max-jobs', '--max-rss', '--max-usd', '--mcp-even-if-plugin', '--mcp-only', '--migrate-only', '--model', '--multimodal', '--name', '--name-only', '--name-status', '--near-symbol', '--nice', '--no', '--no-capture', '--no-cron', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-hooks', '--no-mutate', '--no-verify', '--oauth-client-secret', '--older-than', '--once', '--others', '--overwrite', '--parallel', '--params', '--pat-file', '--path', '--pattern', '--pending', '--pglite', '--phase', '--pid-file', '--porcelain', '--port', '--priority', '--probe-pglite', '--progress-interval', '--progress-json', '--project', '--pure', '--push-only', '--query', '--queue', '--quiet', '--rebase', '--rebuild-rollup', '--refresh', '--refresh-unqualified', '--regenerate', '--remediate', '--remediation-plan', '--remove', '--repo', '--reranker', '--reset', '--resolve', '--restore-only', '--resume', '--retarget', '--review-lower', '--rollback', '--scope', '--scopes', '--set', '--short', '--show-current', '--show-toplevel', '--since', '--skills-dir', '--skip-bare-tweet', '--skip-failed', '--skip-urls', '--skip-verify', '--slugs', '--source', '--source-id', '--stale', '--stats', '--status', '--strategy', '--strict', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--target', '--target-score', '--thin', '--timeout', '--to', '--token', '--token-name', '--token-ttl', '--top-k', '--type', '--undo', '--undo-wave', '--unsafe-bypass-dream-guard', '--unset-all', '--untracked-files', '--url', '--use-captured-snapshot', '--user-hooks', '--verbose', '--verify', '--version', '--window', '--with-calibration', '--workers', '--yes'],
|
||||
'dream': ['--against', '--aliases', '--all', '--allow-regression', '--anchor', '--asof', '--audit-rejects', '--background', '--batch', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--budget-usd', '--budget-usd-answer', '--budget-usd-retrieval', '--by-type', '--by-type-floor', '--cancel-unmatched', '--code', '--committed-baseline', '--compare', '--compile', '--concurrent', '--ctx-size', '--cycles', '--date', '--detail', '--dim', '--dimensions', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--expansion', '--explain', '--fast', '--federated', '--fix', '--fixtures', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--force-retry', '--format', '--from', '--from-db', '--from-pages', '--gold', '--harness', '--help', '--http', '--include-holdout', '--include-null-signature', '--input', '--install', '--json', '--judge-model', '--justification', '--keyword-only', '--lang', '--limit', '--llm', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--max-tokens', '--max-usd', '--mcp-only', '--min-recall', '--mode', '--model', '--models', '--modes', '--multimodal', '--name', '--name-only', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-llm', '--no-mutate', '--no-trajectory', '--once', '--out', '--output', '--output-dir', '--parallel', '--path', '--pattern', '--pending', '--pglite', '--phase', '--priority', '--progress-interval', '--progress-json', '--pull', '--quiet', '--receipt-dir', '--reconcile-queue', '--remediate', '--repo', '--reranking', '--reset', '--resolve', '--restore-only', '--resume-from', '--retrieval-only', '--rounds', '--rubric-version', '--save', '--seed', '--short', '--show-toplevel', '--since', '--skip-replay', '--slot-a-model', '--slot-b-model', '--slot-c-model', '--slug', '--slug-prefix', '--source', '--source-guard', '--source-id', '--stale', '--suite', '--suites', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--take', '--task', '--thin', '--threshold', '--timeout', '--to', '--token-ttl', '--top-k', '--undo', '--unsafe-bypass-dream-guard', '--update-baseline', '--verify', '--version', '--window', '--yes'],
|
||||
'doctor': ['--ab', '--abbrev-ref', '--abi', '--abort', '--aliases', '--all', '--allow-shell-jobs', '--allow-unverified-remote', '--auto', '--auto-fix', '--auto-update', '--background', '--batch', '--brain', '--brain-wide-max-cost-usd', '--branch', '--break-lock', '--build-index', '--by-mention', '--by-type', '--cached', '--check', '--column', '--compile', '--concurrency', '--confidence', '--confirm', '--content-audit', '--count', '--days', '--delete-brain', '--detach', '--detail', '--diff-filter', '--dim', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--env', '--exclude-standard', '--exclusive', '--explain', '--fast', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--force-schema', '--force-sunset-target', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--gbrain-bin', '--get', '--git-dir', '--git-path', '--grant-types', '--harness', '--health-interval', '--help', '--history', '--home', '--http', '--id', '--ignore-env-override', '--ignore-missing-key', '--include-flagged', '--include-frontmatter', '--include-null-signature', '--include-pseudo', '--index-audit', '--init', '--input', '--install', '--is-inside-work-tree', '--job-isolation', '--jq', '--json', '--lang', '--limit', '--local', '--locks', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-crashes', '--max-jobs', '--max-rss', '--max-usd', '--mcp-even-if-plugin', '--mcp-only', '--migrate-only', '--model', '--multimodal', '--name', '--name-only', '--name-status', '--near-symbol', '--nice', '--no', '--no-capture', '--no-cron', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-hooks', '--no-mutate', '--no-verify', '--oauth-client-secret', '--older-than', '--once', '--others', '--overwrite', '--parallel', '--params', '--pat-file', '--path', '--pattern', '--pending', '--pglite', '--phase', '--pid-file', '--porcelain', '--port', '--priority', '--probe-pglite', '--progress-interval', '--progress-json', '--project', '--pure', '--push-only', '--query', '--queue', '--quiet', '--rebase', '--rebuild-rollup', '--refresh', '--refresh-unqualified', '--regenerate', '--remediate', '--remediation-plan', '--remove', '--repo', '--reranker', '--reset', '--resolve', '--restore-only', '--resume', '--retarget', '--review-lower', '--rollback', '--scope', '--scopes', '--set', '--short', '--show-current', '--show-toplevel', '--since', '--skills-dir', '--skip-bare-tweet', '--skip-failed', '--skip-urls', '--skip-verify', '--slugs', '--source', '--source-id', '--stale', '--stats', '--status', '--strategy', '--strict', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--target', '--target-score', '--thin', '--timeout', '--to', '--token', '--token-name', '--token-ttl', '--top-k', '--type', '--undo-wave', '--unsafe-bypass-dream-guard', '--unset-all', '--untracked-files', '--url', '--use-captured-snapshot', '--user-hooks', '--verbose', '--verify', '--version', '--window', '--with-calibration', '--workers', '--yes'],
|
||||
'dream': ['--against', '--aliases', '--all', '--allow-regression', '--anchor', '--asof', '--audit-rejects', '--background', '--batch', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--budget-usd', '--budget-usd-answer', '--budget-usd-retrieval', '--by-type', '--by-type-floor', '--cancel-unmatched', '--code', '--committed-baseline', '--compare', '--compile', '--concurrent', '--ctx-size', '--cycles', '--date', '--detail', '--dim', '--dimensions', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--expansion', '--explain', '--fast', '--federated', '--fix', '--fixtures', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--force-retry', '--format', '--from', '--from-db', '--from-pages', '--gold', '--harness', '--help', '--http', '--include-holdout', '--include-null-signature', '--input', '--install', '--json', '--judge-model', '--justification', '--keyword-only', '--lang', '--limit', '--llm', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--max-tokens', '--max-usd', '--mcp-only', '--min-recall', '--mode', '--model', '--models', '--modes', '--multimodal', '--name', '--name-only', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-llm', '--no-mutate', '--no-trajectory', '--once', '--out', '--output', '--output-dir', '--parallel', '--path', '--pattern', '--pending', '--pglite', '--phase', '--priority', '--progress-interval', '--progress-json', '--pull', '--quiet', '--receipt-dir', '--reconcile-queue', '--remediate', '--repo', '--reranking', '--reset', '--resolve', '--restore-only', '--resume-from', '--retrieval-only', '--rounds', '--rubric-version', '--save', '--seed', '--short', '--show-toplevel', '--since', '--skip-replay', '--slot-a-model', '--slot-b-model', '--slot-c-model', '--slug', '--slug-prefix', '--source', '--source-guard', '--source-id', '--stale', '--suite', '--suites', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--take', '--task', '--thin', '--threshold', '--timeout', '--to', '--token-ttl', '--top-k', '--unsafe-bypass-dream-guard', '--update-baseline', '--verify', '--version', '--window', '--yes'],
|
||||
'edges-backfill': ['--aliases', '--all', '--all-sources', '--brain', '--concurrency', '--federated', '--help', '--include-null-signature', '--json', '--max-age', '--max-chunks', '--max-cost-usd', '--no-extract', '--no-federated', '--older-than', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--source-guard', '--stale', '--supersessions', '--thin', '--timeout', '--workers'],
|
||||
'embed': ['--aliases', '--all', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--catch-up', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--lang', '--markdown', '--max-age', '--max-cost-usd', '--model', '--multimodal', '--name', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--pace', '--pace-max-concurrency', '--parallel', '--path', '--pattern', '--pending', '--pglite', '--prefix', '--priority', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--restore-only', '--serial', '--slugs', '--source', '--stale', '--supabase', '--supersessions', '--surface', '--symbol-kind', '--thin', '--timeout', '--to', '--token-ttl', '--version'],
|
||||
'enrich': ['--aliases', '--all', '--all-sources', '--allow-empty', '--apply', '--asof', '--auto', '--background', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--budget-usd-per-day', '--by-mention', '--clone-dir', '--code', '--concurrency', '--confirm-destructive', '--content', '--date', '--days', '--detail', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--entities', '--explain', '--fast', '--federated', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--judge-model', '--kind', '--lang', '--limit', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--max-usd', '--min-context', '--mode', '--model', '--multimodal', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--offset', '--older-than', '--order', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--reenrich-after', '--remediate', '--reset', '--resolve', '--restore-only', '--resume', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--source-id', '--stale', '--stats', '--supersessions', '--surface', '--symbol-kind', '--thin', '--thin-threshold', '--timeout', '--to', '--token-ttl', '--trusted-extraction', '--types', '--url', '--url-managed', '--version', '--with-db', '--workers', '--yes'],
|
||||
'enrich': ['--aliases', '--all', '--all-sources', '--allow-empty', '--apply', '--asof', '--auto', '--background', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--budget-usd-per-day', '--by-mention', '--clone-dir', '--code', '--concurrency', '--confirm-destructive', '--content', '--date', '--days', '--detail', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--entities', '--explain', '--fast', '--federated', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--judge-model', '--kind', '--lang', '--limit', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--max-usd', '--min-context', '--mode', '--model', '--multimodal', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--offset', '--older-than', '--order', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--reenrich-after', '--remediate', '--reset', '--resolve', '--restore-only', '--resume', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--source-id', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--symbol-kind', '--thin', '--thin-threshold', '--timeout', '--to', '--token-ttl', '--trusted-extraction', '--types', '--url', '--url-managed', '--version', '--with-db', '--workers', '--yes'],
|
||||
'eval': ['--ab-relational', '--against', '--aliases', '--all', '--allow-regression', '--background', '--baseline', '--batch', '--brain', '--brain-wide-max-cost-usd', '--budget-usd', '--budget-usd-answer', '--budget-usd-retrieval', '--committed-baseline', '--compare', '--compare-limit', '--concurrent', '--config-a', '--config-b', '--corpus', '--cycles', '--days', '--dedup-cosine', '--dedup-max-per-page', '--dedup-type-ratio', '--dim', '--dimensions', '--distance-min', '--embedder', '--embedding-dimensions', '--embedding-model', '--expand', '--explain', '--fast', '--fixtures', '--follow', '--force', '--from-capture', '--from-db', '--from-pages', '--gold', '--grounding-min', '--harness', '--help', '--http', '--include-holdout', '--include-null-signature', '--input', '--json', '--judge', '--justification', '--k', '--limit', '--llm', '--max-pair-chars', '--max-tokens', '--max-usd', '--md', '--metric', '--min-recall', '--mode', '--model', '--models', '--modes', '--multimodal', '--name', '--no', '--no-cache', '--no-embed', '--no-embedding', '--no-expand', '--no-extract', '--no-llm', '--older-than', '--out', '--output', '--output-dir', '--parallel', '--pattern', '--pending', '--progress-interval', '--progress-json', '--qrels', '--queries-file', '--query', '--questions', '--quiet', '--receipt-dir', '--refresh-cache', '--remediate', '--reset', '--resolve', '--rrf-k', '--rubric-version', '--runs', '--sampling', '--save', '--seed', '--severity', '--short', '--show-toplevel', '--since', '--skip-replay', '--slot-a-model', '--slot-b-model', '--slot-c-model', '--slug', '--slug-prefix', '--source', '--stale', '--strategy', '--strict', '--suite', '--suites', '--supersessions', '--surface', '--task', '--thin', '--threshold', '--threshold-expected-top1', '--threshold-first-relevant-hit', '--threshold-jaccard', '--threshold-latency-multiplier', '--threshold-recall-at-k', '--threshold-top1', '--timeout', '--to', '--token-ttl', '--tool', '--top-k', '--top-regressions', '--until', '--update-baseline', '--usefulness-min', '--verbose', '--version', '--with-code-intel', '--yes'],
|
||||
'export': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dir', '--explain', '--federated', '--fix', '--follow', '--help', '--include-null-signature', '--json', '--lang', '--markdown', '--multimodal', '--near-symbol', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--restore-only', '--slug-prefix', '--source', '--source-guard', '--stale', '--supersessions', '--symbol-kind', '--thin', '--timeout', '--type'],
|
||||
'extract': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--by-mention', '--catch-up', '--code', '--concurrency', '--dir', '--dry-run', '--explain', '--federated', '--follow', '--from-meetings', '--help', '--include-frontmatter', '--include-null-signature', '--infer-dates', '--json', '--kind', '--lang', '--markdown', '--max-age', '--max-cost-usd', '--multimodal', '--name-status', '--near-symbol', '--ner', '--no-extract', '--no-federated', '--older-than', '--pack', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--restore-only', '--run-id', '--since', '--slug', '--source', '--source-guard', '--source-id', '--stale', '--strategy', '--supersessions', '--symbol-kind', '--thin', '--timeout', '--type', '--verbose', '--workers', '--yes'],
|
||||
'extract-conversation-facts': ['--aliases', '--all', '--all-sources', '--background', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--by-mention', '--clone-dir', '--code', '--concurrency', '--confirm-destructive', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fix', '--follow', '--force', '--force-break-lock', '--help', '--include-null-signature', '--json', '--judge-model', '--lang', '--limit', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--model', '--multimodal', '--near-symbol', '--no', '--no-embed', '--no-embedding', '--no-extract', '--older-than', '--override-disabled', '--path', '--pattern', '--pending', '--pglite', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--restore-only', '--segment-limit', '--session', '--since', '--sleep', '--slug', '--source', '--source-id', '--stale', '--supabase', '--supersessions', '--symbol-kind', '--thin', '--timeout', '--to', '--types', '--url', '--url-managed', '--version', '--workers', '--yes'],
|
||||
'features': ['--aliases', '--all', '--auto-fix', '--background', '--batch-size', '--brain', '--by-mention', '--catch-up', '--concurrency', '--dir', '--explain', '--from-meetings', '--help', '--include-frontmatter', '--include-null-signature', '--infer-dates', '--json', '--kind', '--ner', '--no-extract', '--pace', '--pace-max-concurrency', '--pack', '--path', '--pattern', '--pending', '--priority', '--progress-json', '--quiet', '--repo', '--reset', '--resolve', '--run-id', '--since', '--slugs', '--source', '--source-id', '--stale', '--supersessions', '--thin', '--type', '--verbose', '--workers'],
|
||||
'files': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dry-run', '--explain', '--fast', '--follow', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--no-embedding', '--no-extract', '--no-pointer', '--page', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--retry-failed', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--type', '--yes'],
|
||||
'forget': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'forget': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'founder': ['--aliases', '--all', '--brain', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--mcp-only', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--since', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--until'],
|
||||
'friction': ['--agent', '--base', '--brain', '--compare', '--help', '--hint', '--json', '--kind', '--message', '--no-redact', '--phase', '--redact', '--run-id', '--severity', '--source', '--transcript-path', '--transcripts'],
|
||||
'frontmatter': ['--aliases', '--all', '--allow-catch-all', '--brain', '--cached', '--diff-filter', '--dry-run', '--exclude-standard', '--fast', '--fix', '--force', '--from-pages', '--get', '--help', '--http', '--include-catch-all', '--include-null-signature', '--json', '--name-only', '--name-status', '--no-embedding', '--no-extract', '--no-verify', '--others', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--strategy', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--uninstall', '--write-back'],
|
||||
@@ -61,24 +61,24 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'init': ['--all', '--brain', '--chat-model', '--check', '--ctx-size', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--entity', '--expansion-model', '--fast', '--flag', '--force', '--from-pages', '--grant-types', '--help', '--http', '--issuer-url', '--json', '--judge-model', '--key', '--mcp-only', '--mcp-url', '--migrate-only', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--non-interactive', '--oauth-client-id', '--oauth-client-secret', '--path', '--pglite', '--provenance', '--reranking', '--schema-pack', '--scopes', '--skip-embed-check', '--source', '--stale', '--supabase', '--surface', '--to', '--token-ttl', '--touchpoint', '--url', '--version', '--yes'],
|
||||
'integrations': ['--auto', '--brain', '--dry-run', '--embeddings', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--no-embedding', '--overwrite', '--refresh', '--reranking', '--source', '--surface', '--target', '--token-ttl'],
|
||||
'integrity': ['--aliases', '--all', '--auto', '--backend', '--background', '--brain', '--brain-wide-max-cost-usd', '--check', '--confidence', '--cost', '--dry-run', '--explain', '--fast', '--follow', '--force', '--fresh', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--limit', '--no-embedding', '--no-extract', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--review-lower', '--skip-bare-tweet', '--skip-urls', '--source', '--stale', '--supabase', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--type', '--url'],
|
||||
'jobs': ['--abbrev-ref', '--aliases', '--all', '--allow-empty', '--allow-protected', '--allow-shell-jobs', '--apply', '--asof', '--auto', '--auto-fix', '--auto-with-prompt', '--background', '--backoff-delay', '--backoff-jitter', '--backoff-type', '--batch', '--batch-size', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--break-lock', '--budget-usd', '--budget-usd-per-day', '--by-mention', '--by-type', '--cached', '--catch-up', '--check', '--cli-path', '--cluster', '--cluster-errors', '--code', '--concurrency', '--confidence', '--confirm-destructive', '--content', '--date', '--days', '--delay', '--detach', '--diff-filter', '--dim', '--dimensions', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--empty', '--entities', '--exclude', '--exclude-standard', '--explain', '--fast', '--federated', '--federated-read', '--ff-only', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--hard-deadline', '--health-interval', '--held-out', '--help', '--http', '--idempotency-key', '--image', '--include-frontmatter', '--include-gitignored', '--include-null-signature', '--infer-dates', '--inject-bootstrap', '--inline', '--input', '--install', '--interval', '--is-ancestor', '--job-id', '--job-isolation', '--json', '--kind', '--lang', '--limit', '--lock', '--lock-duration-ms', '--markdown', '--max-age', '--max-attempts', '--max-cost-usd', '--max-crashes', '--max-rss', '--max-runtime-min', '--max-sources', '--max-stalled', '--max-usd', '--max-waiting', '--mcp-only', '--migrate-only', '--min-context', '--missing-path', '--mode', '--model', '--multimodal', '--name-only', '--name-status', '--near-symbol', '--ner', '--nice', '--no', '--no-auto-embed', '--no-embed', '--no-embedding', '--no-extract', '--no-federate', '--no-federated', '--no-gpg-sign', '--no-hard-deadline', '--no-inject', '--no-mutate', '--no-pull', '--no-renames', '--no-schema-pack', '--no-verify', '--no-worker', '--non-interactive', '--now', '--offset', '--older-than', '--once', '--order', '--orphan', '--others', '--output', '--override-disabled', '--pace', '--pace-max-concurrency', '--pack', '--parallel', '--params', '--path', '--pattern', '--pending', '--phase', '--pid-file', '--priority', '--progress-interval', '--progress-json', '--queue', '--quiet', '--redact-secrets', '--reenrich-after', '--refresh-cache', '--refresh-ms', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--respect-gitignore', '--restore-only', '--resume', '--retry-failed', '--review-lower', '--run-id', '--save', '--segment-limit', '--serial', '--session', '--session-id', '--short', '--show-toplevel', '--sigkill-rescue', '--since', '--skip-bare-tweet', '--skip-failed', '--skip-urls', '--sleep', '--slug', '--slugs', '--source', '--source-id', '--src-subpath', '--stale', '--stats', '--status', '--strategy', '--supersessions', '--surface', '--swap-only', '--symbol-kind', '--target', '--target-score', '--thin', '--thin-threshold', '--timeout', '--timeout-ms', '--to', '--token-ttl', '--trusted-extraction', '--type', '--types', '--uninstall', '--unsafe-bypass-dream-guard', '--url', '--user', '--verbose', '--verify', '--version', '--watch', '--wedge-rescue', '--with-db', '--workers', '--yes'],
|
||||
'jobs': ['--abbrev-ref', '--aliases', '--all', '--allow-empty', '--allow-protected', '--allow-shell-jobs', '--apply', '--asof', '--auto', '--auto-fix', '--auto-with-prompt', '--background', '--backoff-delay', '--backoff-jitter', '--backoff-type', '--batch', '--batch-size', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--break-lock', '--budget-usd', '--budget-usd-per-day', '--by-mention', '--by-type', '--cached', '--catch-up', '--check', '--cli-path', '--cluster', '--cluster-errors', '--code', '--concurrency', '--confidence', '--confirm-destructive', '--content', '--date', '--days', '--delay', '--detach', '--diff-filter', '--dim', '--dimensions', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--empty', '--entities', '--exclude', '--exclude-standard', '--explain', '--fast', '--federated', '--federated-read', '--ff-only', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--hard-deadline', '--health-interval', '--held-out', '--help', '--http', '--idempotency-key', '--image', '--include-frontmatter', '--include-gitignored', '--include-null-signature', '--infer-dates', '--inject-bootstrap', '--inline', '--input', '--install', '--interval', '--is-ancestor', '--job-id', '--job-isolation', '--json', '--kind', '--lang', '--limit', '--lock', '--lock-duration-ms', '--markdown', '--max-age', '--max-attempts', '--max-cost-usd', '--max-crashes', '--max-rss', '--max-runtime-min', '--max-sources', '--max-stalled', '--max-usd', '--max-waiting', '--mcp-only', '--migrate-only', '--min-context', '--missing-path', '--mode', '--model', '--multimodal', '--name-only', '--name-status', '--near-symbol', '--ner', '--nice', '--no', '--no-auto-embed', '--no-embed', '--no-embedding', '--no-extract', '--no-federate', '--no-federated', '--no-gpg-sign', '--no-hard-deadline', '--no-inject', '--no-mutate', '--no-pull', '--no-renames', '--no-schema-pack', '--no-verify', '--no-worker', '--non-interactive', '--now', '--offset', '--older-than', '--once', '--order', '--orphan', '--others', '--output', '--override-disabled', '--pace', '--pace-max-concurrency', '--pack', '--parallel', '--params', '--path', '--pattern', '--pending', '--phase', '--pid-file', '--priority', '--progress-interval', '--progress-json', '--queue', '--quiet', '--redact-secrets', '--reenrich-after', '--refresh-cache', '--refresh-ms', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--respect-gitignore', '--restore-only', '--resume', '--retry-failed', '--review-lower', '--run-id', '--save', '--segment-limit', '--serial', '--session', '--session-id', '--short', '--show-toplevel', '--sigkill-rescue', '--since', '--skip-bare-tweet', '--skip-failed', '--skip-urls', '--sleep', '--slug', '--slugs', '--source', '--source-id', '--src-subpath', '--stale', '--stats', '--status', '--stdin', '--strategy', '--supersessions', '--surface', '--swap-only', '--symbol-kind', '--target', '--target-score', '--thin', '--thin-threshold', '--timeout', '--timeout-ms', '--to', '--token-ttl', '--trusted-extraction', '--type', '--types', '--uninstall', '--unsafe-bypass-dream-guard', '--url', '--user', '--verbose', '--verify', '--version', '--watch', '--wedge-rescue', '--with-db', '--workers', '--yes'],
|
||||
'lint': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dry-run', '--exclude', '--explain', '--fast', '--fix', '--follow', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--no-embedding', '--no-extract', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl'],
|
||||
'lsd': ['--brain', '--force-resume', '--help', '--json', '--judge-model', '--limit', '--list-runs', '--max-cost', '--max-far-set', '--max-ideas-per-judge-call', '--no-save', '--resume', '--retry-judge', '--save', '--source', '--strict-budget', '--yes'],
|
||||
'maintain': ['--aliases', '--all', '--background', '--brain', '--break-lock', '--by-mention', '--catch-up', '--concurrency', '--content-audit', '--count', '--detach', '--dim', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fast', '--fix', '--force', '--force-retry', '--force-schema', '--from-meetings', '--full', '--help', '--include-flagged', '--include-frontmatter', '--include-null-signature', '--index-audit', '--infer-dates', '--input', '--json', '--kind', '--lang', '--locks', '--markdown', '--max-cost', '--max-cost-usd', '--max-jobs', '--max-rss', '--max-usd', '--migrate-only', '--multimodal', '--near-symbol', '--ner', '--nice', '--no-extract', '--no-mutate', '--older-than', '--once', '--pack', '--parallel', '--params', '--path', '--pattern', '--pending', '--pglite', '--phase', '--pid-file', '--porcelain', '--probe-pglite', '--progress-json', '--query', '--queue', '--quiet', '--rebuild-rollup', '--regenerate', '--remediate', '--remediation-plan', '--reset', '--resolve', '--restore-only', '--resume', '--run-id', '--safe', '--scope', '--since', '--skills-dir', '--skip-failed', '--slugs', '--source', '--source-id', '--stale', '--status', '--supabase', '--supersessions', '--symbol-kind', '--target', '--target-score', '--thin', '--to', '--top-k', '--type', '--unsafe-bypass-dream-guard', '--url', '--verbose', '--window', '--workers', '--yes'],
|
||||
'migrate': ['--ab', '--aliases', '--all', '--auto-update', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--build-index', '--by-mention', '--catch-up', '--compile', '--days', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--exclusive', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--force-schema', '--force-sunset-target', '--from-meetings', '--from-pages', '--help', '--history', '--http', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--lang', '--locks', '--markdown', '--max-age', '--model', '--multimodal', '--name', '--near-symbol', '--nice', '--no', '--no-embed', '--no-embedding', '--no-extract', '--non-interactive', '--pace', '--pace-max-concurrency', '--parallel', '--path', '--pattern', '--pending', '--phase', '--prefix', '--priority', '--progress-interval', '--progress-json', '--quiet', '--refresh-unqualified', '--remediate', '--reranker', '--reranking', '--reset', '--resolve', '--restore-only', '--resume', '--retarget', '--rollback', '--skip-verify', '--slugs', '--source', '--stale', '--status', '--supersessions', '--surface', '--symbol-kind', '--thin', '--timeout', '--to', '--token-ttl', '--undo', '--undo-wave', '--url', '--use-captured-snapshot', '--version', '--with-calibration', '--yes'],
|
||||
'models': ['--aliases', '--all', '--brain', '--ctx-size', '--detail', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--help', '--include-null-signature', '--json', '--judge-model', '--model', '--multimodal', '--no', '--no-embed', '--no-extract', '--pattern', '--pending', '--reranking', '--reset', '--resolve', '--skip', '--source', '--stale', '--supersessions', '--thin', '--to', '--undo', '--version'],
|
||||
'migrate': ['--ab', '--aliases', '--all', '--auto-update', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--build-index', '--by-mention', '--catch-up', '--compile', '--days', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--exclusive', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-retry', '--force-schema', '--force-sunset-target', '--from-meetings', '--from-pages', '--help', '--history', '--http', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--lang', '--locks', '--markdown', '--max-age', '--model', '--multimodal', '--name', '--near-symbol', '--nice', '--no', '--no-embed', '--no-embedding', '--no-extract', '--non-interactive', '--pace', '--pace-max-concurrency', '--parallel', '--path', '--pattern', '--pending', '--phase', '--prefix', '--priority', '--progress-interval', '--progress-json', '--quiet', '--refresh-unqualified', '--remediate', '--reranker', '--reranking', '--reset', '--resolve', '--restore-only', '--resume', '--retarget', '--rollback', '--skip-verify', '--slugs', '--source', '--stale', '--status', '--supersessions', '--surface', '--symbol-kind', '--thin', '--timeout', '--to', '--token-ttl', '--undo-wave', '--url', '--use-captured-snapshot', '--version', '--with-calibration', '--yes'],
|
||||
'models': ['--aliases', '--all', '--brain', '--ctx-size', '--detail', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--help', '--include-null-signature', '--json', '--judge-model', '--model', '--multimodal', '--no', '--no-embed', '--no-extract', '--pattern', '--pending', '--reranking', '--reset', '--resolve', '--skip', '--source', '--stale', '--supersessions', '--thin', '--to', '--version'],
|
||||
'mounts': ['--alias', '--brain', '--cache', '--database-path', '--database-url', '--db-path', '--db-url', '--engine', '--explain', '--help', '--id', '--json', '--lang', '--lock', '--markdown', '--mcp-url', '--multimodal', '--near-symbol', '--path', '--restore-only', '--skills-dir', '--source', '--stale', '--symbol-kind', '--thin', '--verbose'],
|
||||
'notability-eval': ['--aliases', '--all', '--brain', '--dim', '--embedding-dimensions', '--embedding-model', '--help', '--in', '--include-null-signature', '--json', '--model', '--multimodal', '--no', '--no-embed', '--no-extract', '--out', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--skip-llm', '--source', '--stale', '--supersessions', '--target-high', '--target-low', '--target-medium', '--thin', '--to', '--version'],
|
||||
'onboard': ['--aliases', '--all', '--allow-empty', '--allow-protected', '--apply', '--asof', '--auto', '--auto-with-prompt', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--check', '--content', '--date', '--days', '--entities', '--explain', '--federated', '--file', '--follow', '--from-pages', '--help', '--history', '--http', '--image', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mode', '--multimodal', '--no-extract', '--offset', '--params', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--remediation-plan', '--reset', '--resolve', '--resume', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stale', '--stats', '--supersessions', '--surface', '--target-score', '--thin', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'onboard': ['--aliases', '--all', '--allow-empty', '--allow-protected', '--apply', '--asof', '--auto', '--auto-with-prompt', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--check', '--content', '--date', '--days', '--entities', '--explain', '--federated', '--file', '--follow', '--from-pages', '--help', '--history', '--http', '--image', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mode', '--multimodal', '--no-extract', '--offset', '--params', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--remediation-plan', '--reset', '--resolve', '--resume', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--target-score', '--thin', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'orphans': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--count', '--explain', '--follow', '--help', '--include-null-signature', '--include-pseudo', '--json', '--no-extract', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin', '--timeout'],
|
||||
'pages': ['--aliases', '--all', '--brain', '--dry-run', '--help', '--include-null-signature', '--json', '--no-extract', '--older-than', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin'],
|
||||
'pglite-repair': ['--brain', '--break-lock', '--dry-rnu', '--dry-run', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--no-embedding', '--path', '--quiet', '--source', '--surface', '--token-ttl', '--yes'],
|
||||
'post-upgrade': ['--all', '--apply-clean-hunks', '--brain', '--check', '--code', '--compile', '--concurrency', '--detail', '--dim', '--embedding-dimensions', '--embedding-model', '--fast', '--ff-only', '--flag', '--force', '--force-all', '--force-orchestrator', '--force-retry', '--force-schema', '--format', '--from-pages', '--help', '--host-dir', '--http', '--inject-bootstrap', '--inline', '--install', '--interval', '--json', '--limit', '--list', '--markdown', '--max-rss', '--migrate-only', '--migration', '--mode', '--model', '--multimodal', '--name-only', '--no', '--no-autopilot-install', '--no-embed', '--no-embedding', '--no-inject', '--no-worker', '--non-interactive', '--now', '--path', '--pglite', '--quiet', '--repo', '--reset', '--since', '--skills-dir', '--skip-verify', '--source', '--stale', '--status', '--supabase', '--surface', '--swap-only', '--target', '--to', '--token-ttl', '--uninstall', '--user', '--verbose', '--verify', '--version', '--workers', '--yes'],
|
||||
'protocol': ['--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--json', '--kind', '--limit', '--max-usd', '--mode', '--multimodal', '--no-embedding', '--offset', '--path', '--progress-interval', '--progress-json', '--quiet', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stats', '--surface', '--synthesize', '--target', '--timeout', '--token', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'providers': ['--brain', '--ctx-size', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--reranking', '--source', '--surface', '--to', '--token-ttl', '--touchpoint', '--version'],
|
||||
'protocol': ['--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--json', '--kind', '--limit', '--max-usd', '--mode', '--multimodal', '--no-embedding', '--offset', '--path', '--progress-interval', '--progress-json', '--quiet', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stats', '--stdin', '--surface', '--synthesize', '--target', '--timeout', '--token', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'providers': ['--brain', '--ctx-size', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--reranking', '--source', '--surface', '--to', '--token-ttl', '--touchpoint', '--version', '--yes'],
|
||||
'publish': ['--accent', '--bg', '--border', '--brain', '--card-bg', '--code-bg', '--error', '--fg', '--help', '--json', '--link', '--muted', '--out', '--password', '--source', '--title'],
|
||||
'quarantine': ['--aliases', '--all', '--apply', '--brain', '--code', '--compile', '--explain', '--fast', '--fix', '--force', '--force-rechunk', '--from-pages', '--help', '--http', '--include-flagged', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--multimodal', '--near-symbol', '--no-embed', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--restore-only', '--source', '--stale', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl'],
|
||||
'recall': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'quarantine': ['--aliases', '--all', '--apply', '--brain', '--code', '--compile', '--explain', '--fast', '--fix', '--force', '--force-rechunk', '--from-pages', '--help', '--http', '--include-flagged', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--multimodal', '--near-symbol', '--no-embed', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--restore-only', '--source', '--source-id', '--stale', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl'],
|
||||
'recall': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'reconcile-links': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dry-run', '--explain', '--follow', '--help', '--include-frontmatter', '--include-null-signature', '--json', '--name-status', '--no-extract', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--source', '--stale', '--strategy', '--supersessions', '--thin', '--timeout', '--type'],
|
||||
'reindex': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--code', '--compile', '--concurrency', '--cost-estimate', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--max-age', '--max-cost-usd', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--no-extract', '--older-than', '--path', '--pattern', '--pending', '--pglite', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--source', '--stale', '--supabase', '--supersessions', '--surface', '--thin', '--timeout', '--to', '--token-ttl', '--version', '--workers', '--yes'],
|
||||
'reindex-code': ['--abi', '--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--chunker-debug', '--code', '--compile', '--concurrency', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fix', '--follow', '--force', '--force-rechunk', '--help', '--include-null-signature', '--json', '--judge-model', '--lang', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-runtime', '--model', '--multimodal', '--no', '--no-embed', '--no-extract', '--older-than', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--serial', '--source', '--stale', '--supersessions', '--thin', '--timeout', '--to', '--version', '--workers', '--yes'],
|
||||
@@ -90,7 +90,7 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'report': ['--brain', '--content', '--dir', '--help', '--json', '--source', '--title', '--type'],
|
||||
'repos': ['--abbrev-ref', '--abort', '--aliases', '--all', '--all-sources', '--allow-unverified-remote', '--brain', '--branch', '--break-lock', '--cached', '--clone-dir', '--compile', '--confirm-destructive', '--count', '--detect', '--diff-filter', '--dry-run', '--exclude-standard', '--explain', '--fast', '--federated', '--file', '--fix', '--force', '--force-break-lock', '--format', '--from-pages', '--full', '--get', '--git-dir', '--git-path', '--github-repo', '--help', '--http', '--include-null-signature', '--include-warns', '--is-inside-work-tree', '--json', '--keep-storage', '--lang', '--local', '--markdown', '--max-age', '--max-cost-usd', '--message', '--multimodal', '--name', '--name-only', '--name-status', '--near-symbol', '--no-cron', '--no-embedding', '--no-extract', '--no-federate', '--no-federated', '--no-harden', '--no-verify', '--others', '--params', '--pat-file', '--path', '--pattern', '--pending', '--porcelain', '--push-only', '--quiet', '--rebase', '--repo', '--reset', '--resolve', '--restore-only', '--secret', '--set', '--short', '--show-toplevel', '--source', '--source-guard', '--source-id', '--stale', '--status', '--strategy', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl', '--unset-all', '--url', '--url-managed', '--yes'],
|
||||
'resolvers': ['--auto', '--backend', '--brain', '--cost', '--help', '--json', '--source'],
|
||||
'retrieval-upgrade': ['--aliases', '--all', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--catch-up', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-sunset-target', '--from-pages', '--help', '--http', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--max-age', '--model', '--multimodal', '--name', '--nice', '--no', '--no-embed', '--no-embedding', '--no-extract', '--non-interactive', '--pace', '--pace-max-concurrency', '--parallel', '--pattern', '--pending', '--prefix', '--priority', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reranker', '--reranking', '--reset', '--resolve', '--resume', '--retarget', '--slugs', '--source', '--stale', '--status', '--supersessions', '--surface', '--thin', '--timeout', '--to', '--token-ttl', '--undo', '--version', '--yes'],
|
||||
'retrieval-upgrade': ['--aliases', '--all', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--catch-up', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--embeddings', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-sunset-target', '--from-pages', '--help', '--http', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--max-age', '--model', '--multimodal', '--name', '--nice', '--no', '--no-embed', '--no-embedding', '--no-extract', '--non-interactive', '--pace', '--pace-max-concurrency', '--parallel', '--pattern', '--pending', '--prefix', '--priority', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reranker', '--reranking', '--reset', '--resolve', '--resume', '--retarget', '--slugs', '--source', '--stale', '--status', '--supersessions', '--surface', '--thin', '--timeout', '--to', '--token-ttl', '--version', '--yes'],
|
||||
'routing-eval': ['--brain', '--fix', '--help', '--json', '--llm', '--skills-dir', '--source', '--strict', '--verbose'],
|
||||
'salience': ['--aliases', '--all', '--brain', '--days', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--kind', '--limit', '--mcp-only', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--slug-prefix', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl'],
|
||||
'schema': ['--alias', '--aliases', '--all', '--apply', '--as-filing-rules', '--brain', '--dims', '--expert', '--expert-routing', '--extractable', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--inverse', '--json', '--kind', '--no-embedding', '--no-extract', '--pack', '--page-type', '--pattern', '--pending', '--prefix', '--primitive', '--reset', '--resolve', '--schema-pack', '--since', '--source', '--source-id', '--stale', '--supersessions', '--surface', '--target-type', '--thin', '--to', '--token-ttl', '--with-db'],
|
||||
@@ -106,10 +106,11 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'storage': ['--aliases', '--all', '--brain', '--federated', '--fix', '--help', '--include-null-signature', '--json', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--source-guard', '--stale', '--supersessions', '--thin', '--to'],
|
||||
'sweep': ['--aliases', '--all', '--batch-limit', '--brain', '--budget-ms', '--help', '--include-null-signature', '--json', '--no-extract', '--once', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin', '--timeout'],
|
||||
'sync': ['--abbrev-ref', '--abi', '--abort', '--aliases', '--all', '--all-sources', '--allow-empty', '--asof', '--auto', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--branch', '--break-lock', '--by-mention', '--cached', '--catch-up', '--clone-dir', '--code', '--column', '--compile', '--concurrency', '--confirm-destructive', '--content-audit', '--count', '--depth', '--detach', '--diff-filter', '--dim', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--empty', '--exclude', '--exclude-standard', '--explain', '--fast', '--federated', '--ff-only', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--force-retry', '--force-schema', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--git-path', '--hard-deadline', '--help', '--http', '--include-flagged', '--include-frontmatter', '--include-gitignored', '--include-null-signature', '--index-audit', '--infer-dates', '--interval', '--is-ancestor', '--json', '--kind', '--lang', '--lock', '--locks', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-jobs', '--max-rss', '--max-sources', '--max-usd', '--migrate-only', '--missing-path', '--model', '--multimodal', '--name', '--name-only', '--name-status', '--ner', '--nice', '--no-auto-embed', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-gpg-sign', '--no-hard-deadline', '--no-pull', '--no-recurse-submodules', '--no-renames', '--no-schema-pack', '--no-verify', '--object-format', '--older-than', '--orphan', '--others', '--overwrite', '--pace', '--pace-max-concurrency', '--pack', '--parallel', '--params', '--path', '--pattern', '--pending', '--pglite', '--phase', '--pid-file', '--porcelain', '--prefix', '--priority', '--probe-pglite', '--progress-interval', '--progress-json', '--query', '--queue', '--quiet', '--rebase', '--rebuild-rollup', '--refresh', '--regenerate', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--respect-gitignore', '--restore-only', '--resume', '--retry-failed', '--run-id', '--save', '--scope', '--serial', '--short', '--show-toplevel', '--since', '--skills-dir', '--skip-failed', '--slug', '--slugs', '--source', '--source-guard', '--source-id', '--src-subpath', '--stale', '--status', '--stdin', '--strategy', '--supabase', '--supersessions', '--surface', '--target', '--target-score', '--thin', '--timeout', '--to', '--token-ttl', '--top-k', '--type', '--url', '--url-managed', '--verbose', '--verify', '--watch', '--window', '--workers', '--yes'],
|
||||
'takes': ['--aliases', '--all', '--brain', '--bucket-size', '--by', '--claim', '--dir', '--domain', '--dry-run', '--evidence', '--expired', '--fast', '--federated', '--force', '--from-pages', '--help', '--holder', '--http', '--include-covered', '--include-null-signature', '--json', '--kind', '--limit', '--max-pages', '--no-embedding', '--no-extract', '--no-federated', '--outcome', '--path', '--pattern', '--pending', '--quality', '--refresh', '--repo', '--reset', '--resolve', '--restore-only', '--row', '--since', '--slugs', '--sort', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--token-ttl', '--unit', '--until', '--value', '--weight', '--who', '--yes'],
|
||||
'takes': ['--aliases', '--all', '--brain', '--bucket-size', '--by', '--claim', '--dir', '--domain', '--dry-run', '--evidence', '--expired', '--fast', '--federated', '--force', '--from-pages', '--help', '--holder', '--http', '--include-covered', '--include-null-signature', '--json', '--kind', '--limit', '--max-pages', '--no-embedding', '--no-extract', '--no-federated', '--outcome', '--path', '--pattern', '--pending', '--quality', '--repo', '--reset', '--resolve', '--restore-only', '--row', '--since', '--slugs', '--sort', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--token-ttl', '--unit', '--until', '--value', '--weight', '--who', '--yes'],
|
||||
'think': ['--aliases', '--all', '--anchor', '--brain', '--calibration-holder', '--explain', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--max-usd', '--mcp-only', '--model', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--rounds', '--save', '--since', '--source', '--stale', '--supersessions', '--surface', '--take', '--thin', '--timeout', '--token-ttl', '--until', '--with-calibration'],
|
||||
'transcripts': ['--aliases', '--all', '--all-discovery', '--background', '--brain', '--brain-wide-max-cost-usd', '--by-mention', '--code', '--compile', '--days', '--dry-run', '--embed', '--explain', '--facts', '--fast', '--federated', '--follow', '--force', '--format', '--from-pages', '--full', '--help', '--http', '--include-null-signature', '--json', '--limit', '--markdown', '--max-cost-usd', '--no-embedding', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--restore-only', '--since', '--slug', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl'],
|
||||
'upgrade': ['--all', '--apply-clean-hunks', '--brain', '--check', '--code', '--compile', '--concurrency', '--detail', '--dim', '--embedding-dimensions', '--embedding-model', '--fast', '--ff-only', '--flag', '--force', '--force-all', '--force-orchestrator', '--force-retry', '--force-schema', '--format', '--from-pages', '--help', '--host-dir', '--http', '--inject-bootstrap', '--inline', '--install', '--interval', '--json', '--limit', '--list', '--markdown', '--max-rss', '--migrate-only', '--migration', '--mode', '--model', '--multimodal', '--name-only', '--no', '--no-autopilot-install', '--no-embed', '--no-embedding', '--no-inject', '--no-worker', '--non-interactive', '--now', '--path', '--pglite', '--quiet', '--repo', '--reset', '--since', '--skills-dir', '--skip-verify', '--source', '--stale', '--status', '--supabase', '--surface', '--swap-only', '--target', '--to', '--token-ttl', '--uninstall', '--user', '--verbose', '--verify', '--version', '--workers', '--yes'],
|
||||
'watch': ['--aliases', '--all', '--brain', '--fast', '--federated', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--max-pages', '--min-confidence', '--no-embedding', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--source-guard', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--token-ttl', '--window-turns'],
|
||||
'ze-switch': ['--aliases', '--all', '--brain', '--confirm-reembed', '--dim', '--dry-run', '--force', '--help', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--markdown', '--no-extract', '--non-interactive', '--pattern', '--pending', '--reset', '--resolve', '--resume', '--source', '--stale', '--supersessions', '--thin', '--to', '--undo', '--yes'],
|
||||
'whoknows': ['--aliases', '--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--detail', '--entities', '--explain', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--include-null-signature', '--json', '--kind', '--lang', '--limit', '--markdown', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--near-symbol', '--no-embedding', '--no-extract', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--reset', '--resolve', '--restore-only', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stale', '--stats', '--stdin', '--supersessions', '--surface', '--symbol-kind', '--thin', '--timeout', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'ze-switch': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--confirm-reembed', '--dim', '--dry-run', '--explain', '--follow', '--force', '--force-sunset-target', '--help', '--ignore-env-override', '--ignore-missing-key', '--include-null-signature', '--json', '--markdown', '--no-extract', '--non-interactive', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reranker', '--reset', '--resolve', '--resume', '--source', '--stale', '--supersessions', '--thin', '--timeout', '--to', '--undo', '--yes'],
|
||||
};
|
||||
|
||||
@@ -1191,6 +1191,10 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
|
||||
// stops claiming "Nothing in gbrain reads this" for a key the resolver
|
||||
// reads on every unqualified call.
|
||||
'sources.default',
|
||||
// Alias/undeclared explicit-type warnings at sync/import (default on).
|
||||
// Read by performSync + runImport summary aggregation; 'false'/'0'/'off'
|
||||
// silences both surfaces (schema lint rules stay active).
|
||||
'schema.type_warnings',
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -1211,6 +1215,12 @@ export const KNOWN_CONFIG_KEY_PREFIXES: readonly string[] = [
|
||||
'autopilot.', // autopilot.nightly_quality_probe.*, autopilot.auto_drain.* (#1685)
|
||||
'chronicle.', // chronicle.tz + future Life Chronicle knobs (#2390)
|
||||
'self_upgrade.', // v0.42 self-upgrade (mode, quiet_hours, state)
|
||||
// Queue admission control (per-name sub-keys):
|
||||
// minions.coalesce_params.<name>, minions.ttl_waiting_hours.<name>,
|
||||
// minions.quota_max_waiting.<name>, plus the one-time
|
||||
// minions.ttl_notice_shown flag. Booleans via the canonical truthiness
|
||||
// parser; numeric 0 disables.
|
||||
'minions.',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -24,6 +24,7 @@ import { randomUUID } from 'node:crypto';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { PhaseResult, PhaseError } from '../cycle.ts';
|
||||
import { MinionQueue } from '../minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../minions/admission.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, MinionJobStatus, SubagentHandlerData } from '../minions/types.ts';
|
||||
import { serializeMarkdown } from '../markdown.ts';
|
||||
@@ -211,9 +212,20 @@ export async function runPhasePatterns(
|
||||
timeout_ms: budgets.timeoutMs,
|
||||
queue: childQueueName,
|
||||
};
|
||||
const job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
let job: Awaited<ReturnType<typeof queue.add>>;
|
||||
try {
|
||||
job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only): a
|
||||
// rejected submit is a recorded phase SKIP, never a phase crash — the
|
||||
// next cycle retries once the backlog drains.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
return skipped('admission_quota', e.message);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Drain this phase's private child queue inline so the parent observes
|
||||
// the terminal state instead of polling waitForCompletion until
|
||||
|
||||
@@ -52,6 +52,7 @@ import { parseLlmJson } from '../llm-json.ts';
|
||||
import type { BrainEngine, DreamVerdict, TriageSegment } from '../engine.ts';
|
||||
import type { PhaseResult, PhaseError } from '../cycle.ts';
|
||||
import { MinionQueue } from '../minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../minions/admission.ts';
|
||||
import { reconnectAfterConnectionError } from '../minions/reconnect.ts';
|
||||
import { isRetryableConnError } from '../retry-matcher.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
@@ -821,7 +822,15 @@ export async function runPhaseSynthesize(
|
||||
}
|
||||
}
|
||||
|
||||
// Admission-quota latch: once a submit is rejected, every later transcript
|
||||
// this run would be rejected too — record one skip per remaining file
|
||||
// without hammering the queue.
|
||||
let quotaHit = false;
|
||||
for (const t of worthProcessing) {
|
||||
if (quotaHit) {
|
||||
skipReports.push({ filePath: t.filePath, reason: 'admission_quota: submission stopped this run' });
|
||||
continue;
|
||||
}
|
||||
const hash16 = t.contentHash.slice(0, 16);
|
||||
const hash6 = t.contentHash.slice(0, 6);
|
||||
|
||||
@@ -911,6 +920,12 @@ export async function runPhaseSynthesize(
|
||||
? `anthropic:${config.model}`
|
||||
: config.model;
|
||||
const triageVerdict = pass.byPath.get(t.filePath);
|
||||
// Fresh (non-coalesced) chunk submissions for THIS transcript — rolled
|
||||
// back if a later chunk hits the admission quota, so a transcript never
|
||||
// half-synthesizes while its skip report claims it was skipped
|
||||
// (adversarial finding). Coalesced rows are another run's bookkeeping
|
||||
// and must not be cancelled.
|
||||
const transcriptFreshIds: number[] = [];
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const childData: SubagentHandlerData = {
|
||||
prompt: buildSynthesisPrompt(
|
||||
@@ -940,12 +955,36 @@ export async function runPhaseSynthesize(
|
||||
timeout_ms: config.subagentTimeoutMs,
|
||||
queue: childQueueName,
|
||||
};
|
||||
let child = await queue.add(
|
||||
'subagent',
|
||||
childData as unknown as Record<string, unknown>,
|
||||
submitOpts,
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
let child: Awaited<ReturnType<typeof queue.add>>;
|
||||
try {
|
||||
child = await queue.add(
|
||||
'subagent',
|
||||
childData as unknown as Record<string, unknown>,
|
||||
submitOpts,
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only):
|
||||
// a rejected submit is a recorded phase skip, never a phase crash —
|
||||
// same posture as daily_cap_reached. The quota won't clear mid-run,
|
||||
// so stop submitting for this run entirely. Roll back this
|
||||
// transcript's already-submitted fresh chunks first: draining a
|
||||
// partial chunk set would write partial pages for a transcript the
|
||||
// skip report says was skipped.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
for (const id of transcriptFreshIds) {
|
||||
try { await queue.cancelJob(id); } catch { /* best-effort rollback */ }
|
||||
const idx = childIds.indexOf(id);
|
||||
if (idx >= 0) childIds.splice(idx, 1);
|
||||
jobRawSource.delete(id);
|
||||
chunkInfo.delete(id);
|
||||
}
|
||||
skipReports.push({ filePath: t.filePath, reason: `admission_quota: ${e.message}` });
|
||||
quotaHit = true;
|
||||
break;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
// Self-heal (#4152 C1): an idempotency-coalesced row still `waiting`
|
||||
// in a FOREIGN dream-inline-* queue was stranded by a previously
|
||||
// killed/timed-out run — no worker will ever claim it, and waiting on
|
||||
@@ -971,7 +1010,10 @@ export async function runPhaseSynthesize(
|
||||
);
|
||||
}
|
||||
}
|
||||
if (child.coalesced !== true) submittedToday++;
|
||||
if (child.coalesced !== true) {
|
||||
submittedToday++;
|
||||
transcriptFreshIds.push(child.id);
|
||||
}
|
||||
childIds.push(child.id);
|
||||
jobRawSource.set(child.id, t.filePath);
|
||||
if (isChunked) {
|
||||
|
||||
@@ -86,6 +86,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'facts_extraction_health',
|
||||
'facts_health',
|
||||
'frontmatter_integrity',
|
||||
'malformed_path_pages',
|
||||
'grade_confidence_drift',
|
||||
'graph_coverage',
|
||||
'graph_signals_coverage',
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
* Provider-agnostic embedding migration (#3390).
|
||||
*
|
||||
* `gbrain migrate embeddings --to <provider:model>` re-embeds a brain onto
|
||||
* any configured provider — the forward path off a sunsetting provider that
|
||||
* `ze-switch` (ZE-only target) and `ze-switch --undo` (needs a snapshot fresh
|
||||
* installs don't have) cannot cover.
|
||||
* any configured provider — the ONE forward path off a sunsetting provider
|
||||
* (the retired ze-switch is a refusal/redirect shim that points here).
|
||||
*
|
||||
* This module is the v0.47 SURVIVOR: the migration primitives live HERE
|
||||
* (runSchemaTransition, transitionDimPinnedColumn, detectEnvOverride and the
|
||||
|
||||
+82
-12
@@ -4,12 +4,13 @@ import { createHash } from 'crypto';
|
||||
import { marked } from 'marked';
|
||||
import type { BrainEngine, FileSpec } from './engine.ts';
|
||||
import { parseMarkdown } from './markdown.ts';
|
||||
import { classifyStoredType } from './schema-pack/type-usage.ts';
|
||||
import { chunkText } from './chunkers/recursive.ts';
|
||||
import { chunkCodeText, chunkCodeTextFull, detectCodeLanguage, CHUNKER_VERSION } from './chunkers/code.ts';
|
||||
import { findChunkForOffset } from './chunkers/edge-extractor.ts';
|
||||
import { extractCodeRefs, imageOfCandidates } from './link-extraction.ts';
|
||||
import { embedBatch, embedMultimodal, currentEmbeddingSignature } from './embedding.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath, hasMalformedPathSegment } from './sync.ts';
|
||||
import type { ChunkInput, PageInput, PageType } from './types.ts';
|
||||
import { computeEffectiveDate } from './effective-date.ts';
|
||||
import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts';
|
||||
@@ -235,6 +236,21 @@ export interface ImportResult {
|
||||
flagged?: boolean;
|
||||
/** Which flag tier fired, when `flagged`. */
|
||||
flag_reason?: 'markup_heavy' | 'oversized';
|
||||
/**
|
||||
* Machine-readable skip class for status='skipped' rows that must NOT be
|
||||
* treated as failures. 'malformed_path' = the FILENAME contains bracket or
|
||||
* control characters (never importable; rename the file) — sync counts these
|
||||
* in its malformed summary and keeps them OUT of failedFiles / the failure
|
||||
* ledger so they can never gate bookmark advancement.
|
||||
*/
|
||||
skip_reason?: 'malformed_path';
|
||||
/**
|
||||
* Advisory (schema.type_warnings): the page's explicit frontmatter `type:`
|
||||
* is an alias of a canonical pack type or undeclared in the pack. The type
|
||||
* is stored literally either way; sync/import aggregate these once per
|
||||
* distinct type per run.
|
||||
*/
|
||||
type_warning?: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string };
|
||||
}
|
||||
|
||||
const MAX_FILE_SIZE = 5_000_000; // 5MB
|
||||
@@ -295,7 +311,7 @@ export async function importFromContent(
|
||||
* Callers thread this from `loadActivePack(ctx)` once per command —
|
||||
* NEVER per file inside sync (codex perf finding #7).
|
||||
*/
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> };
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
|
||||
/**
|
||||
* v0.39.3.0 provenance write-through (WARN-8). When set, threaded to
|
||||
* `tx.putPage` so the page's `source_kind`, `source_uri`,
|
||||
@@ -583,7 +599,11 @@ export async function importFromContent(
|
||||
// #1035: fetch the existing page BEFORE the hash compute so (a) the type
|
||||
// preservation below participates in the hash (a no-op re-put stays a
|
||||
// hash-match skip) and (b) the hash short-circuit below reuses this row.
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
// Scoped to the exact (source_id, slug) row the writes below target —
|
||||
// engine.putPage defaults to 'default' when sourceId is unset, so the read
|
||||
// mirrors that default instead of matching the slug in ANY source (the
|
||||
// unscoped-check/scoped-write bug class).
|
||||
const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
|
||||
// #2044: remote get_page intentionally strips private facts rows. A
|
||||
// documented get_page -> edit -> put_page round-trip can therefore arrive
|
||||
@@ -614,6 +634,22 @@ export async function importFromContent(
|
||||
parsed.type = existing.type;
|
||||
}
|
||||
|
||||
// Alias-footgun visibility: an explicit frontmatter `type:` that is an
|
||||
// ALIAS of a canonical pack type (or entirely undeclared) is stored
|
||||
// literally and never re-normalized — different agents can silently file
|
||||
// the same concept under different types/directories. Classify it here
|
||||
// (once per file, aggregated once per type per run by sync/import) so the
|
||||
// misroute class is loud. Purely advisory: the type is still stored as-is.
|
||||
let typeWarning: ImportResult['type_warning'];
|
||||
if (parsed.typeExplicit === true && opts.activePack) {
|
||||
const cls = classifyStoredType(parsed.type, opts.activePack);
|
||||
if (cls.kind === 'alias_of') {
|
||||
typeWarning = { kind: 'alias_of', type: parsed.type, canonical: cls.canonical, directory: cls.directory };
|
||||
} else if (cls.kind === 'undeclared') {
|
||||
typeWarning = { kind: 'undeclared', type: parsed.type };
|
||||
}
|
||||
}
|
||||
|
||||
const HASH_EPHEMERAL_FRONTMATTER_KEYS = [
|
||||
'captured_at',
|
||||
'ingested_at',
|
||||
@@ -647,7 +683,7 @@ export async function importFromContent(
|
||||
};
|
||||
|
||||
if (existing?.content_hash === hash && !opts.forceRechunk) {
|
||||
return { slug, status: 'skipped', chunks: 0, parsedPage };
|
||||
return { slug, status: 'skipped', chunks: 0, parsedPage, ...(typeWarning ? { type_warning: typeWarning } : {}) };
|
||||
}
|
||||
|
||||
// v0.41.13 (#1309) — identity-based cross-slug dedup pre-check.
|
||||
@@ -689,7 +725,7 @@ export async function importFromContent(
|
||||
}
|
||||
if (dup && dup.slug !== slug) {
|
||||
// Look up the duplicate page so we can compare frontmatter.id.
|
||||
const dupPage = await engine.getPage(dup.slug, sourceId ? { sourceId } : undefined);
|
||||
const dupPage = await engine.getPage(dup.slug, { sourceId: sourceId ?? 'default' });
|
||||
const dupFmId = (dupPage?.frontmatter as Record<string, unknown> | undefined)?.id;
|
||||
const dupFmIdStr = typeof dupFmId === 'string' && dupFmId.length > 0 ? dupFmId : null;
|
||||
const sameExternalId = fmIdStr !== null && dupFmIdStr === fmIdStr;
|
||||
@@ -820,7 +856,7 @@ export async function importFromContent(
|
||||
// caller's sourceId so writes target (sourceId, slug) rather than the
|
||||
// schema DEFAULT — required for multi-source brains; harmless ('default')
|
||||
// for single-source callers.
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
const txOpts = { sourceId: sourceId ?? 'default' };
|
||||
await engine.transaction(async (tx) => {
|
||||
if (existing) await tx.createVersion(slug, txOpts);
|
||||
|
||||
@@ -1000,6 +1036,7 @@ export async function importFromContent(
|
||||
parsedPage,
|
||||
...(pageQuarantined ? { quarantined: true } : {}),
|
||||
...(pageFlagged ? { flagged: true, flag_reason: pageFlagReason } : {}),
|
||||
...(typeWarning ? { type_warning: typeWarning } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1021,7 +1058,7 @@ async function verifyPageReadable(
|
||||
sourceId: string | undefined,
|
||||
caller: string,
|
||||
): Promise<void> {
|
||||
const readBack = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
const readBack = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!readBack) {
|
||||
// Log to ingest_log before throwing so the failure is durable and
|
||||
// agent-inspectable, not just a transient stderr message.
|
||||
@@ -1087,7 +1124,7 @@ export async function importFromFile(
|
||||
* `parseMarkdown` uses pack-driven type inference. Load ONCE per command;
|
||||
* never per file (codex perf finding #7).
|
||||
*/
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> };
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
|
||||
} = {},
|
||||
): Promise<ImportResult> {
|
||||
// Defense-in-depth: reject symlinks before reading content.
|
||||
@@ -1103,6 +1140,27 @@ export async function importFromFile(
|
||||
|
||||
let content = readFileSync(filePath, 'utf-8');
|
||||
|
||||
// Defense-in-depth for callers that bypass the sync/import classifiers
|
||||
// (direct importFromFile, reindex, capture paths): a malformed filename is
|
||||
// never importable. Checked BEFORE the code dispatch and BEFORE any YAML
|
||||
// parsing (codex re-review P2: a control-char code path returned through
|
||||
// importCodeFile, and broken-YAML junk returned a parse error instead of
|
||||
// this informational skip). hasMalformedPathSegment is markdown-scoped for
|
||||
// brackets, so legit bracketed code dirs (`app/[id]/`) still dispatch;
|
||||
// control characters reject on every path. skip_reason marks this as
|
||||
// informational so sync's failure gate never counts it.
|
||||
if (hasMalformedPathSegment(relativePath)) {
|
||||
return {
|
||||
slug: '',
|
||||
status: 'skipped',
|
||||
skip_reason: 'malformed_path',
|
||||
chunks: 0,
|
||||
error:
|
||||
`Path "${relativePath}" contains bracket or control characters and ` +
|
||||
`cannot be imported. Rename the file to import it.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Route code files through the code import path
|
||||
if (isCodeFilePath(relativePath)) {
|
||||
return importCodeFile(engine, relativePath, content, {
|
||||
@@ -1149,6 +1207,10 @@ export async function importFromFile(
|
||||
// parsed.slug is `frontmatter.slug || inferSlug(filePath)` where inferSlug
|
||||
// falls back to slugifyPath(). So parsed.slug.length > 0 with empty
|
||||
// expectedSlug = frontmatter provided one; both empty = no usable slug.
|
||||
// (The malformed-path defense runs earlier, before the code dispatch —
|
||||
// slugifyPath must never see a junk filename: it would STRIP the brackets
|
||||
// and mint a plausible-looking slug, the exact mechanism that polluted
|
||||
// search in the poisoned-path incident.)
|
||||
const expectedSlug = slugifyPath(relativePath);
|
||||
let resolvedSlug = expectedSlug;
|
||||
let usedFrontmatterFallback = false;
|
||||
@@ -1246,7 +1308,7 @@ export async function importCodeFile(
|
||||
const lang = detectCodeLanguage(relativePath) || 'unknown';
|
||||
const title = `${relativePath} (${lang})`;
|
||||
const sourceId = opts.sourceId;
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
const txOpts = { sourceId: sourceId ?? 'default' };
|
||||
// PostgreSQL text columns reject U+0000 even though source files may
|
||||
// legitimately contain it inside string/regex fixtures. Preserve a visible,
|
||||
// searchable representation instead of dropping the entire code page.
|
||||
@@ -1279,7 +1341,11 @@ export async function importCodeFile(
|
||||
.update(JSON.stringify({ title, type: 'code', content, lang, chunker_version: CHUNKER_VERSION }))
|
||||
.digest('hex');
|
||||
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
// Scoped to the exact (source_id, slug) row the writes below target —
|
||||
// engine.putPage defaults to 'default' when sourceId is unset, so the read
|
||||
// mirrors that default instead of matching the slug in ANY source (the
|
||||
// unscoped-check/scoped-write bug class).
|
||||
const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!opts.force && existing?.content_hash === hash) {
|
||||
return { slug, status: 'skipped', chunks: 0 };
|
||||
}
|
||||
@@ -1317,7 +1383,7 @@ export async function importCodeFile(
|
||||
// OpenAI API. Order matters: our chunk_index is semantic (tree-sitter
|
||||
// order), so a matching (chunk_index, text_hash) means a verbatim
|
||||
// preserved symbol.
|
||||
const existingChunks = existing ? await engine.getChunks(slug, sourceId ? { sourceId } : undefined) : [];
|
||||
const existingChunks = existing ? await engine.getChunks(slug, { sourceId: sourceId ?? 'default' }) : [];
|
||||
const existingByKey = new Map<string, typeof existingChunks[number]>();
|
||||
for (const ec of existingChunks) {
|
||||
existingByKey.set(`${ec.chunk_index}:${ec.chunk_text}`, ec);
|
||||
@@ -1775,7 +1841,11 @@ export async function importImageFile(
|
||||
// and slugifyPath would already preserve it). Recompute with the file
|
||||
// extension preserved so the page slug is stable + collision-free.
|
||||
const imageSlug = relativePath.replace(/[\\\/]/g, '/').toLowerCase();
|
||||
const sourceOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
// Scoped to the exact (source_id, slug) row the write targets — same
|
||||
// unscoped-check/scoped-write fix as importFromContent/importCodeFile
|
||||
// above (the variable-bound ternary shape evaded the CI guard's inline
|
||||
// heuristic; caught by adversarial review).
|
||||
const sourceOpts = { sourceId: opts.sourceId ?? 'default' };
|
||||
const linkOpts = opts.sourceId
|
||||
? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId, originSourceId: opts.sourceId }
|
||||
: undefined;
|
||||
|
||||
@@ -1053,7 +1053,9 @@ export function makeResolver(
|
||||
// (unwrapped by unwrapWikilink) that name a real page the strict regex
|
||||
// could not reach and whose full-path fuzzy score is below threshold.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) {
|
||||
const page = await engine.getPage(trimmed);
|
||||
// Same source scope as the basename index above (#972): a wikilink in
|
||||
// source A must not resolve to a same-slug page in source B.
|
||||
const page = await engine.getPage(trimmed, opts.sourceId ? { sourceId: opts.sourceId } : undefined); // gbrain-allow-unscoped-getpage: read-only wikilink resolution; unscoped-when-no-source is the documented single-source behavior
|
||||
if (page) {
|
||||
cache.set(cacheKey, trimmed);
|
||||
return trimmed;
|
||||
@@ -1065,7 +1067,7 @@ export function makeResolver(
|
||||
for (const hint of hints) {
|
||||
if (!hint) continue;
|
||||
const candidate = `${hint}/${slugified}`;
|
||||
const page = await engine.getPage(candidate);
|
||||
const page = await engine.getPage(candidate, opts.sourceId ? { sourceId: opts.sourceId } : undefined); // gbrain-allow-unscoped-getpage: read-only wikilink resolution; unscoped-when-no-source is the documented single-source behavior
|
||||
if (page) {
|
||||
cache.set(cacheKey, candidate);
|
||||
return candidate;
|
||||
|
||||
@@ -76,6 +76,45 @@ export function coerceFrontmatterString(v: unknown): string {
|
||||
return String(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte offset of the first character AFTER the closing frontmatter fence —
|
||||
* i.e. where the body starts and where a body-only editor may safely operate
|
||||
* without ever touching frontmatter bytes.
|
||||
*
|
||||
* Fence semantics mirror collectValidationErrors exactly (the canonical
|
||||
* definition): leading blank lines are allowed before the opener, fences are
|
||||
* matched with trim() so CRLF line endings (`---\r`) count. Returns 0 when the
|
||||
* file has no frontmatter at all (first non-empty line is not `---`) — there
|
||||
* is no fence to protect, the whole file is body. Returns 0 for an UNCLOSED
|
||||
* fence too; callers that must not edit such files should pre-validate with
|
||||
* parseMarkdown({validate:true}) and treat MISSING_CLOSE as a blocker (the
|
||||
* backlinks fixer does).
|
||||
*/
|
||||
export function frontmatterBodyOffset(content: string): number {
|
||||
const lines = content.split('\n');
|
||||
|
||||
let offset = 0;
|
||||
let i = 0;
|
||||
// Skip leading blank lines.
|
||||
for (; i < lines.length; i++) {
|
||||
if (lines[i].trim().length > 0) break;
|
||||
offset += lines[i].length + 1;
|
||||
}
|
||||
if (i >= lines.length) return 0; // empty / whitespace-only file
|
||||
if (lines[i].trim() !== '---') return 0; // no frontmatter
|
||||
|
||||
offset += lines[i].length + 1; // consume the opening fence line
|
||||
for (i = i + 1; i < lines.length; i++) {
|
||||
const isLast = i === lines.length - 1;
|
||||
const lineLen = lines[i].length + (isLast ? 0 : 1);
|
||||
offset += lineLen;
|
||||
if (lines[i].trim() === '---') {
|
||||
return Math.min(offset, content.length);
|
||||
}
|
||||
}
|
||||
return 0; // unclosed fence — no safe body offset
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a markdown file with YAML frontmatter into its components.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* Migration-ledger summary — the read-only "what host migrations are
|
||||
* outstanding" view, shared by the get_health op (TODOS:4063: remote agents
|
||||
* detect wedged migrations without SSH-ing to run the doctor) and the
|
||||
* apply-migrations orchestrator (which imports statusForVersion +
|
||||
* compareVersions from here).
|
||||
*
|
||||
* OV4/EV4 (CLI→MCP gap-closure wave): an op must never import
|
||||
* src/commands/apply-migrations.ts — that module statically pulls the whole
|
||||
* migration registry (17 migration modules plus orchestrator machinery). This
|
||||
* module works on VERSION STRINGS ONLY: MIGRATION_VERSIONS below is a plain
|
||||
* string list, pinned against the real registry by
|
||||
* test/migration-ledger.test.ts so it cannot drift silently.
|
||||
*/
|
||||
|
||||
import { loadCompletedMigrations, type CompletedMigrationEntry } from './preferences.ts';
|
||||
|
||||
/**
|
||||
* Version strings of every registered host migration, in registry order.
|
||||
* APPEND HERE when adding a migration module to src/commands/migrations/
|
||||
* (the sync test fails the suite otherwise).
|
||||
*/
|
||||
export const MIGRATION_VERSIONS: readonly string[] = [
|
||||
'0.11.0', '0.12.0', '0.12.2', '0.13.0', '0.13.1', '0.14.0', '0.16.0',
|
||||
'0.18.0', '0.18.1', '0.21.0', '0.22.4', '0.28.0', '0.29.1', '0.31.0',
|
||||
'0.32.2', '0.43.0', '0.46.3',
|
||||
];
|
||||
|
||||
/** Bug 3 attempt cap — consecutive partials before a version counts wedged. */
|
||||
export const MAX_CONSECUTIVE_PARTIALS = 3;
|
||||
|
||||
/**
|
||||
* Compare two semver strings (MAJOR.MINOR.PATCH). Returns -1 / 0 / 1.
|
||||
* Canonical home (moved from src/commands/migrations/index.ts, which
|
||||
* re-exports it); originally extracted from upgrade.ts#isNewerThan.
|
||||
*/
|
||||
export function compareVersions(a: string, b: string): -1 | 0 | 1 {
|
||||
const va = a.split('.').map(n => parseInt(n, 10) || 0);
|
||||
const vb = b.split('.').map(n => parseInt(n, 10) || 0);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const da = va[i] ?? 0;
|
||||
const db = vb[i] ?? 0;
|
||||
if (da > db) return 1;
|
||||
if (da < db) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export type MigrationLedgerStatus = 'complete' | 'partial' | 'pending' | 'wedged';
|
||||
|
||||
/**
|
||||
* Resolved status for a migration version given its ledger entries.
|
||||
*
|
||||
* Semantics (Bug 3 — keep "complete wins" safety):
|
||||
* - If the latest entry is `retry`, the version is pending. This is the
|
||||
* explicit escape hatch written by a forced retry, and it overrides an
|
||||
* earlier `complete` entry without hand-editing the ledger.
|
||||
* - Otherwise, if any entry is `complete`, the version is complete.
|
||||
* - Otherwise, MAX_CONSECUTIVE_PARTIALS trailing partials → wedged.
|
||||
* - Otherwise, any `partial` entry → partial; else pending.
|
||||
*
|
||||
* `complete` never regresses accidentally. A later `partial` append cannot
|
||||
* undo a completed migration; only a trailing, explicit `retry` marker can.
|
||||
*/
|
||||
export function statusForVersion(
|
||||
version: string,
|
||||
byVersion: Map<string, CompletedMigrationEntry[]>,
|
||||
): MigrationLedgerStatus {
|
||||
const entries = byVersion.get(version) ?? [];
|
||||
if (entries.length === 0) return 'pending';
|
||||
const latest = entries[entries.length - 1];
|
||||
if (latest.status === 'retry') return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
let consecutive = 0;
|
||||
for (let i = entries.length - 1; i >= 0; i--) {
|
||||
const e = entries[i];
|
||||
if (e.status === 'partial') consecutive++;
|
||||
else break;
|
||||
}
|
||||
if (consecutive >= MAX_CONSECUTIVE_PARTIALS) return 'wedged';
|
||||
if (entries.some(e => e.status === 'partial')) return 'partial';
|
||||
return 'pending';
|
||||
}
|
||||
|
||||
export function indexCompletedEntries(
|
||||
entries: CompletedMigrationEntry[],
|
||||
): Map<string, CompletedMigrationEntry[]> {
|
||||
const byVersion = new Map<string, CompletedMigrationEntry[]>();
|
||||
for (const e of entries) {
|
||||
const list = byVersion.get(e.version) ?? [];
|
||||
list.push(e);
|
||||
byVersion.set(e.version, list);
|
||||
}
|
||||
return byVersion;
|
||||
}
|
||||
|
||||
export interface MigrationLedgerSummary {
|
||||
/** Registered migrations ≤ installed version with no ledger completion. */
|
||||
pending: string[];
|
||||
/** Started but unfinished (some phases recorded partial). */
|
||||
partial: string[];
|
||||
/** Hit the consecutive-partial cap — needs an explicit forced retry. */
|
||||
wedged: string[];
|
||||
/**
|
||||
* Migrations newer than the installed binary (count only). Future-versioned
|
||||
* migrations are excluded from ALL status buckets regardless of ledger
|
||||
* state (matches the apply-migrations list view); after a binary downgrade
|
||||
* a wedged future migration appears only in this count.
|
||||
*/
|
||||
skipped_future: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarize the host migration ledger for the given installed version.
|
||||
* Filesystem read (the completed-migrations JSONL) — engine-agnostic, which
|
||||
* is why get_health composes this at the op layer rather than growing
|
||||
* BrainEngine.getHealth() in both engines. Version strings only; never
|
||||
* migration internals.
|
||||
*/
|
||||
export function migrationLedgerSummary(installedVersion: string): MigrationLedgerSummary {
|
||||
const byVersion = indexCompletedEntries(loadCompletedMigrations());
|
||||
const summary: MigrationLedgerSummary = { pending: [], partial: [], wedged: [], skipped_future: 0 };
|
||||
for (const version of MIGRATION_VERSIONS) {
|
||||
if (compareVersions(version, installedVersion) > 0) {
|
||||
summary.skipped_future += 1;
|
||||
continue;
|
||||
}
|
||||
const status = statusForVersion(version, byVersion);
|
||||
if (status === 'pending') summary.pending.push(version);
|
||||
else if (status === 'partial') summary.partial.push(version);
|
||||
else if (status === 'wedged') summary.wedged.push(version);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
/**
|
||||
* Admission control for the minion queue — the submit-side half of the
|
||||
* queue-divergence fix (the drain-side pool-starvation half landed in
|
||||
* v0.46.1.0). Three primitives, all ADMISSION-side by design (claim
|
||||
* fairness / lane scheduling is explicitly out of scope, tracked in TODOS):
|
||||
*
|
||||
* 1. PARAM-COALESCING — an identical parentless submit (same name, queue,
|
||||
* owner lane, and payload hash) coalesces onto the newest matching
|
||||
* WAITING row instead of enqueuing a duplicate. Targets the
|
||||
* runaway-producer class: crons re-submitting the same prompt hundreds
|
||||
* of times a day into a queue that drains a fraction of that.
|
||||
* 2. WAITING-TTL — a job still WAITING after N hours is cancelled (via the
|
||||
* canonical cancel path, so parents/aggregators resolve) with an
|
||||
* auditable error_text. At structural divergence (intake >> drain),
|
||||
* FIFO wait exceeds any plausible usefulness horizon — cancelling
|
||||
* visibly beats queueing forever.
|
||||
* 3. NAME-GLOBAL QUOTA — reject (typed error, never a silent coalesce)
|
||||
* submits once a name's TOTAL waiting count across ALL queues reaches
|
||||
* the configured cap. Counts name-globally because fanout producers use
|
||||
* per-run private queues (dream-inline-*): a (name, queue)-scoped count
|
||||
* would reset to zero for every new private queue and never bind.
|
||||
* NO shipped default (user decision D2C) — activates only via config.
|
||||
*
|
||||
* Per-name defaults table pattern follows handler-timeouts.ts. Config keys
|
||||
* (DB plane, registered under the 'minions.' prefix):
|
||||
* minions.coalesce_params.<name> ('false'/'0'/'off' disables; default on
|
||||
* only for names in PARAM_COALESCE_DEFAULT)
|
||||
* minions.ttl_waiting_hours.<name> (number; 0 disables; default only for
|
||||
* names in WAITING_TTL_DEFAULT_HOURS)
|
||||
* minions.quota_max_waiting.<name> (number; no defaults)
|
||||
*
|
||||
* Env kill-switch: GBRAIN_MINIONS_ADMISSION=0 disables all three wholesale
|
||||
* (incident escape hatch — no DB needed).
|
||||
*
|
||||
* All lookups fail OPEN to the defaults tables: an unreadable config must
|
||||
* never block job submission. The first failure per process logs one stderr
|
||||
* warning (a silent fail-open is a silent failure).
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
|
||||
/** Names whose parentless submits coalesce on identical params by default. */
|
||||
export const PARAM_COALESCE_DEFAULT: Readonly<Record<string, boolean>> = {
|
||||
subagent: true,
|
||||
};
|
||||
|
||||
/** Default waiting-TTL hours per name. Absent = no TTL. */
|
||||
export const WAITING_TTL_DEFAULT_HOURS: Readonly<Record<string, number>> = {
|
||||
subagent: 48,
|
||||
};
|
||||
|
||||
/**
|
||||
* Default name-global waiting quotas. EMPTY by design (user decision D2C):
|
||||
* the quota mechanism ships but activates only via
|
||||
* `minions.quota_max_waiting.<name>` config. The DIVERGENT-queue scream in
|
||||
* `jobs stats` / doctor is the default-on protection layer and carries the
|
||||
* opt-in hint.
|
||||
*/
|
||||
export const QUOTA_MAX_WAITING_DEFAULT: Readonly<Record<string, number>> = {};
|
||||
|
||||
/**
|
||||
* Keys excluded from the param hash. ONLY the hash's own storage key:
|
||||
* `__owner_client_id` is deliberately INCLUDED so coalescing never crosses
|
||||
* owner lanes — one OAuth client's submit must not be suppressed by (or
|
||||
* handed a job id owned by) another client.
|
||||
*/
|
||||
export const PARAM_HASH_EXCLUDED_KEYS: ReadonlySet<string> = new Set(['__param_hash']);
|
||||
|
||||
/**
|
||||
* error_text prefix stamped by the waiting-TTL sweep. The 24h-cancellation
|
||||
* surfaces (jobs stats, doctor) LIKE-match on this exact prefix — keep the
|
||||
* sweep's reason string and the consumers' patterns derived from ONE constant
|
||||
* so they can never drift apart.
|
||||
*/
|
||||
export const TTL_REASON_PREFIX = 'waiting_ttl_expired';
|
||||
|
||||
/**
|
||||
* Config flag recording WHEN the one-time waiting-TTL notice was shown
|
||||
* (ISO timestamp; legacy value 'true' = shown at unknown time). Shared by
|
||||
* the worker's warn-before-act gate and the runPostUpgrade banner.
|
||||
*/
|
||||
export const TTL_NOTICE_SHOWN_KEY = 'minions.ttl_notice_shown';
|
||||
|
||||
/**
|
||||
* Grace window between the one-time TTL notice and the first sweep (user
|
||||
* requirement D1A: warn BEFORE acting, with enough time to actually react —
|
||||
* one maintenance tick (~30s) is not a warning, it's a courtesy log line).
|
||||
* Env override is a test seam / incident hatch.
|
||||
*/
|
||||
export const TTL_NOTICE_GRACE_MS_DEFAULT = 60 * 60 * 1000;
|
||||
|
||||
export function ttlNoticeGraceMs(): number {
|
||||
const v = Number(process.env.GBRAIN_MINIONS_TTL_NOTICE_GRACE_MS);
|
||||
return Number.isFinite(v) && v >= 0 ? v : TTL_NOTICE_GRACE_MS_DEFAULT;
|
||||
}
|
||||
|
||||
/** Typed admission rejection — submitters surface the message or record a skip. */
|
||||
export class QueueQuotaExceededError extends Error {
|
||||
readonly code = 'quota_exceeded';
|
||||
constructor(
|
||||
public readonly jobName: string,
|
||||
public readonly waiting: number,
|
||||
public readonly quota: number,
|
||||
) {
|
||||
// The message travels to REMOTE MCP clients via submit_agent — it names
|
||||
// the quota (the caller's admission contract) but NOT the live global
|
||||
// waiting count, which would leak cross-tenant queue depth. Operators get
|
||||
// exact counts locally from 'gbrain jobs stats'; the count stays on the
|
||||
// error object for local consumers/tests.
|
||||
super(
|
||||
`queue admission: '${jobName}' is at its waiting quota (${quota}, all queues). ` +
|
||||
`Drain or cancel backlog first — see 'gbrain jobs stats'. ` +
|
||||
`Tune: gbrain config set minions.quota_max_waiting.${jobName} <n> (raise) or remove the key (disable).`,
|
||||
);
|
||||
this.name = 'QueueQuotaExceededError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical quota-rejection check for submitters (agent fanout, dream
|
||||
* synthesize/patterns, ops). Checks the stable `code` field as well as
|
||||
* instanceof/name so it survives dual-module instances and compiled-binary
|
||||
* boundaries where instanceof can lie.
|
||||
*/
|
||||
export function isQueueQuotaExceededError(e: unknown): e is QueueQuotaExceededError {
|
||||
if (e instanceof QueueQuotaExceededError) return true;
|
||||
if (!(e instanceof Error)) return false;
|
||||
return e.name === 'QueueQuotaExceededError' ||
|
||||
(e as { code?: unknown }).code === 'quota_exceeded';
|
||||
}
|
||||
|
||||
/** Stable stringify: recursively sorts object keys so hash(key order) is invariant. */
|
||||
function stableStringify(v: unknown): string {
|
||||
if (v === null || typeof v !== 'object') return JSON.stringify(v) ?? 'undefined';
|
||||
if (Array.isArray(v)) return `[${v.map(stableStringify).join(',')}]`;
|
||||
const o = v as Record<string, unknown>;
|
||||
const keys = Object.keys(o).sort();
|
||||
return `{${keys.map(k => `${JSON.stringify(k)}:${stableStringify(o[k])}`).join(',')}}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* sha256 over the stable-stringified payload minus PARAM_HASH_EXCLUDED_KEYS.
|
||||
* Node-side (not SQL md5(data::text)) so canonicalization is explicit and
|
||||
* unit-testable, with no reliance on jsonb text-rendering parity across
|
||||
* engines. Stored in data.__param_hash (the `__`-prefixed embedded-metadata
|
||||
* convention, like __owner_client_id); a future algorithm change is
|
||||
* forward-safe — old rows just stop matching, which means "no coalesce".
|
||||
*/
|
||||
export function computeParamHash(data: Record<string, unknown>): string {
|
||||
const filtered: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(data)) {
|
||||
if (PARAM_HASH_EXCLUDED_KEYS.has(k)) continue;
|
||||
filtered[k] = data[k];
|
||||
}
|
||||
return createHash('sha256').update(stableStringify(filtered)).digest('hex');
|
||||
}
|
||||
|
||||
export interface AdmissionPolicy {
|
||||
/** Param-coalescing on for this name (parentless submits only). */
|
||||
coalesceParams: boolean;
|
||||
/** Waiting-TTL in hours; null = no TTL for this name. */
|
||||
ttlWaitingHours: number | null;
|
||||
/** Name-global max waiting; null = no quota for this name. */
|
||||
quotaMaxWaiting: number | null;
|
||||
}
|
||||
|
||||
export function admissionKilled(): boolean {
|
||||
return process.env.GBRAIN_MINIONS_ADMISSION === '0';
|
||||
}
|
||||
|
||||
function isOffValue(v: string): boolean {
|
||||
// Case-insensitive + trimmed: an operator typing 'FALSE' or 'Off' means
|
||||
// OFF — an emergency off-switch that only matches exact lowercase tokens
|
||||
// silently stays ON (structured-review finding).
|
||||
const t = v.trim().toLowerCase();
|
||||
return t === 'false' || t === '0' || t === 'off' || t === 'no';
|
||||
}
|
||||
|
||||
/**
|
||||
* Gate for embedding a job name into a COPY-PASTEABLE command hint
|
||||
* (`gbrain config set minions.…<name> …`). Display sanitization strips
|
||||
* control bytes but keeps shell metacharacters; a name like `x$(cmd)` must
|
||||
* never ride into a hint an operator will paste into a shell. Returns the
|
||||
* name when it is a safe config-key segment, else null (caller renders a
|
||||
* placeholder).
|
||||
*/
|
||||
export function safeConfigSegment(name: string): string | null {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(name) ? name : null;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(v: string | null): number | null {
|
||||
if (v == null || v.trim() === '') return null;
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n) || n <= 0) return null; // 0/garbage = disabled
|
||||
return n;
|
||||
}
|
||||
|
||||
// ~60s in-process cache: add() runs per submission; a config read per submit
|
||||
// would add a query to the hot path for a value that changes at human speed.
|
||||
type CacheEntry = { at: number; policy: AdmissionPolicy };
|
||||
const policyCache = new Map<string, CacheEntry>();
|
||||
const POLICY_CACHE_MS = 60_000;
|
||||
let warnedFailOpen = false;
|
||||
|
||||
/** Test seam: drop the cache so config changes are visible immediately. */
|
||||
export function _resetAdmissionCacheForTest(): void {
|
||||
policyCache.clear();
|
||||
warnedFailOpen = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the admission policy for a job name: config overrides > per-name
|
||||
* defaults tables. Fail-open to defaults on any config error (warn once per
|
||||
* process). Kill-switch returns the all-off policy.
|
||||
*/
|
||||
export async function resolveAdmissionPolicy(engine: BrainEngine, jobName: string): Promise<AdmissionPolicy> {
|
||||
if (admissionKilled()) {
|
||||
return { coalesceParams: false, ttlWaitingHours: null, quotaMaxWaiting: null };
|
||||
}
|
||||
const cached = policyCache.get(jobName);
|
||||
if (cached && Date.now() - cached.at < POLICY_CACHE_MS) return cached.policy;
|
||||
|
||||
const policy: AdmissionPolicy = {
|
||||
coalesceParams: PARAM_COALESCE_DEFAULT[jobName] === true,
|
||||
ttlWaitingHours: WAITING_TTL_DEFAULT_HOURS[jobName] ?? null,
|
||||
quotaMaxWaiting: QUOTA_MAX_WAITING_DEFAULT[jobName] ?? null,
|
||||
};
|
||||
try {
|
||||
const [coalesceV, ttlV, quotaV] = await Promise.all([
|
||||
engine.getConfig(`minions.coalesce_params.${jobName}`),
|
||||
engine.getConfig(`minions.ttl_waiting_hours.${jobName}`),
|
||||
engine.getConfig(`minions.quota_max_waiting.${jobName}`),
|
||||
]);
|
||||
if (coalesceV != null && coalesceV.trim() !== '') {
|
||||
policy.coalesceParams = !isOffValue(coalesceV.trim());
|
||||
}
|
||||
if (ttlV != null && ttlV.trim() !== '') {
|
||||
policy.ttlWaitingHours = parsePositiveNumber(ttlV);
|
||||
}
|
||||
if (quotaV != null && quotaV.trim() !== '') {
|
||||
policy.quotaMaxWaiting = parsePositiveNumber(quotaV);
|
||||
}
|
||||
} catch (e) {
|
||||
if (!warnedFailOpen) {
|
||||
warnedFailOpen = true;
|
||||
console.error(
|
||||
`[minions admission] config read failed (${e instanceof Error ? e.message : String(e)}) — ` +
|
||||
`using built-in defaults (coalesce=${policy.coalesceParams}, ttl=${policy.ttlWaitingHours ?? 'off'}h, ` +
|
||||
`quota=${policy.quotaMaxWaiting ?? 'off'}). Warning prints once per process.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
policyCache.set(jobName, { at: Date.now(), policy });
|
||||
return policy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Names with an active waiting-TTL: defaults ∪ config overrides discovered
|
||||
* via listConfigKeys('minions.ttl_waiting_hours.') when the engine supports
|
||||
* it (optional method — same guard pattern as config.ts's key listing).
|
||||
* Returns name → hours, with 0/garbage-configured names removed.
|
||||
*/
|
||||
export async function resolveTtlNames(engine: BrainEngine): Promise<Map<string, number>> {
|
||||
if (admissionKilled()) return new Map();
|
||||
const out = new Map<string, number>();
|
||||
for (const [name, hours] of Object.entries(WAITING_TTL_DEFAULT_HOURS)) out.set(name, hours);
|
||||
try {
|
||||
const listConfigKeys = (engine as { listConfigKeys?: (prefix: string) => Promise<string[]> }).listConfigKeys;
|
||||
if (typeof listConfigKeys === 'function') {
|
||||
const prefix = 'minions.ttl_waiting_hours.';
|
||||
const keys = await listConfigKeys.call(engine, prefix);
|
||||
for (const key of keys) {
|
||||
const name = key.slice(prefix.length);
|
||||
if (!name) continue;
|
||||
const v = parsePositiveNumber(await engine.getConfig(key));
|
||||
if (v == null) out.delete(name); // configured 0/garbage = disabled
|
||||
else out.set(name, v);
|
||||
}
|
||||
} else {
|
||||
// No listing support: still honor overrides for the DEFAULT names.
|
||||
for (const name of Object.keys(WAITING_TTL_DEFAULT_HOURS)) {
|
||||
const v = await engine.getConfig(`minions.ttl_waiting_hours.${name}`);
|
||||
if (v != null && v.trim() !== '') {
|
||||
const n = parsePositiveNumber(v);
|
||||
if (n == null) out.delete(name);
|
||||
else out.set(name, n);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fail open to the defaults already in `out`.
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count currently-waiting jobs already past their per-name TTL. Shared by
|
||||
* the worker's warn-before-act notice and the runPostUpgrade banner so the
|
||||
* two channels can never disagree on what "affected" means.
|
||||
*/
|
||||
export async function countTtlExpiredWaiting(
|
||||
engine: BrainEngine,
|
||||
ttlNames: Map<string, number>,
|
||||
): Promise<{ total: number; by_name: Record<string, number> }> {
|
||||
const by_name: Record<string, number> = {};
|
||||
let total = 0;
|
||||
for (const [name, hours] of ttlNames) {
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs
|
||||
WHERE name = $1 AND status = 'waiting' AND updated_at < now() - ($2 * interval '1 hour')`,
|
||||
[name, hours],
|
||||
);
|
||||
const n = parseInt(rows[0]?.count ?? '0', 10);
|
||||
by_name[name] = n;
|
||||
total += n;
|
||||
}
|
||||
return { total, by_name };
|
||||
}
|
||||
|
||||
/** Minimal structural dep so this module never imports MinionQueue (which imports us). */
|
||||
export interface WaitingTtlSweeper {
|
||||
handleWaitingTTL(opts?: { maxPerTick?: number }): Promise<{ cancelled: number; by_name: Record<string, number> }>;
|
||||
}
|
||||
|
||||
export interface WaitingTtlTickResult {
|
||||
/**
|
||||
* killed — GBRAIN_MINIONS_ADMISSION=0, nothing done.
|
||||
* notice — first tick: counted affected jobs, persisted the notice
|
||||
* timestamp; caller prints the warning. NOTHING cancelled.
|
||||
* grace — notice shown but the grace window hasn't elapsed; no sweep.
|
||||
* swept — sweep ran (cancelled may be 0).
|
||||
*/
|
||||
phase: 'killed' | 'notice' | 'grace' | 'swept';
|
||||
cancelled: number;
|
||||
by_name: Record<string, number>;
|
||||
/** Populated on phase 'notice': waiting jobs already past their TTL. */
|
||||
affected?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One warn-before-act TTL maintenance tick (user requirement D1A), extracted
|
||||
* from the worker interval so it is directly testable (pattern:
|
||||
* lock-renewal-tick.ts). Semantics:
|
||||
*
|
||||
* flag unset → count + stamp TTL_NOTICE_SHOWN_KEY with an ISO
|
||||
* timestamp, return 'notice' (caller warns; no sweep)
|
||||
* flag = ISO timestamp → sweep only after ttlNoticeGraceMs() has elapsed
|
||||
* since the notice ('grace' until then)
|
||||
* flag = 'true' (legacy / pre-grace releases) → sweep immediately
|
||||
*
|
||||
* The upgrade banner stamps the SAME key with the same ISO form, so an
|
||||
* interactive `gbrain upgrade` starts the clock too — whichever channel
|
||||
* shows the notice first, the operator gets a full grace window to set
|
||||
* `minions.ttl_waiting_hours.<name> 0` before anything is cancelled.
|
||||
*/
|
||||
export async function runWaitingTtlTick(
|
||||
engine: BrainEngine,
|
||||
sweeper: WaitingTtlSweeper,
|
||||
): Promise<WaitingTtlTickResult> {
|
||||
if (admissionKilled()) return { phase: 'killed', cancelled: 0, by_name: {} };
|
||||
const flag = (await engine.getConfig(TTL_NOTICE_SHOWN_KEY))?.trim() ?? '';
|
||||
if (flag === '') {
|
||||
const ttlNames = await resolveTtlNames(engine);
|
||||
const { total } = await countTtlExpiredWaiting(engine, ttlNames);
|
||||
await engine.setConfig(TTL_NOTICE_SHOWN_KEY, new Date().toISOString());
|
||||
return { phase: 'notice', cancelled: 0, by_name: {}, affected: total };
|
||||
}
|
||||
if (flag !== 'true') {
|
||||
const ts = Date.parse(flag);
|
||||
if (Number.isFinite(ts) && Date.now() - ts < ttlNoticeGraceMs()) {
|
||||
return { phase: 'grace', cancelled: 0, by_name: {} };
|
||||
}
|
||||
}
|
||||
const res = await sweeper.handleWaitingTTL();
|
||||
return { phase: 'swept', ...res };
|
||||
}
|
||||
+316
-14
@@ -16,6 +16,14 @@ import type {
|
||||
import { rowToMinionJob, rowToInboxMessage, rowToAttachment } from './types.ts';
|
||||
import { validateAttachment } from './attachments.ts';
|
||||
import { isProtectedJobName } from './protected-names.ts';
|
||||
import {
|
||||
computeParamHash,
|
||||
resolveAdmissionPolicy,
|
||||
resolveTtlNames,
|
||||
QueueQuotaExceededError,
|
||||
PARAM_HASH_EXCLUDED_KEYS,
|
||||
TTL_REASON_PREFIX,
|
||||
} from './admission.ts';
|
||||
import {
|
||||
defaultTimeoutMsFor, HANDLER_DEFAULT_TIMEOUT_MS,
|
||||
defaultLockDurationMsFor, HANDLER_DEFAULT_LOCK_DURATION_MS, clampLockDurationMs,
|
||||
@@ -106,6 +114,8 @@ type CoalesceAuditEvent = {
|
||||
queue: string; name: string; returned_job_id: number;
|
||||
waiting_count?: number; max_waiting?: number;
|
||||
pending_count?: number; max_pending?: number;
|
||||
/** Set when the coalesce matched on an identical payload hash (admission). */
|
||||
param_hash?: string;
|
||||
};
|
||||
|
||||
/** Shared cap-hit coalesce return for the backpressure guards: hydrate the
|
||||
@@ -219,6 +229,53 @@ export class MinionQueue {
|
||||
const delayUntil = opts?.delay ? new Date(Date.now() + opts.delay) : null;
|
||||
const maxSpawnDepth = opts?.max_spawn_depth ?? this.maxSpawnDepth;
|
||||
|
||||
// Admission policy (param-coalescing + name-global quota). Resolved
|
||||
// OUTSIDE the transaction (60s in-process cache; fail-open to defaults).
|
||||
// Parented submits never coalesce: fanout children belong to their
|
||||
// parent's bookkeeping/aggregator — returning some other child would
|
||||
// corrupt child_done accounting. opts.coalesce_params overrides per call.
|
||||
const policy = await resolveAdmissionPolicy(this.engine, jobName);
|
||||
// An EMPTY payload (after excluding the hash key itself) carries no
|
||||
// dedupe signal — two no-param submits are more likely distinct
|
||||
// placeholder/scaffolding jobs than a runaway producer (which always
|
||||
// carries a prompt). Never coalesce those.
|
||||
// A caller-supplied idempotency_key also disables param-coalescing:
|
||||
// producer-owned idempotency is the STRONGER contract ("this exact key
|
||||
// maps to this exact row"), and a param-coalesce hit would return a row
|
||||
// the key was never registered against — a later same-key submit would
|
||||
// then insert fresh and run the work twice (adversarial-review finding).
|
||||
const hashablePayload = Object.keys(data ?? {}).some(k => !PARAM_HASH_EXCLUDED_KEYS.has(k));
|
||||
const coalesceActive =
|
||||
(opts?.coalesce_params ?? policy.coalesceParams) &&
|
||||
!opts?.parent_job_id &&
|
||||
!opts?.idempotency_key &&
|
||||
hashablePayload &&
|
||||
childStatus === 'waiting';
|
||||
let paramHash: string | null = null;
|
||||
if (coalesceActive) {
|
||||
// Execution options are part of the coalescing IDENTITY (codex re-review
|
||||
// P1): identical payloads with different timeout/priority/attempt/
|
||||
// quiet-hours semantics are NOT the same job — coalescing them would
|
||||
// silently hand the second submitter the first's execution contract.
|
||||
// Only DEFINED options fold in (conservative: an explicit value never
|
||||
// coalesces onto an implicit-default row; forward-safe like any hash
|
||||
// input change — old rows just stop matching).
|
||||
const optIdentity: Record<string, unknown> = {};
|
||||
for (const k of ['priority', 'timeout_ms', 'max_attempts', 'quiet_hours', 'lock_duration_ms', 'delay_ms', 'max_stalled'] as const) {
|
||||
const v = (opts as Record<string, unknown> | undefined)?.[k];
|
||||
if (v !== undefined) optIdentity[k] = v;
|
||||
}
|
||||
paramHash = computeParamHash(
|
||||
Object.keys(optIdentity).length > 0
|
||||
? { ...(data ?? {}), __opts_identity: optIdentity }
|
||||
: ((data ?? {}) as Record<string, unknown>),
|
||||
);
|
||||
// Clone rather than mutate the caller's object; the hash rides in the
|
||||
// payload (the __-prefixed embedded-metadata convention) so the SQL
|
||||
// match needs no DDL and `jobs get` shows what matched.
|
||||
data = { ...(data ?? {}), __param_hash: paramHash };
|
||||
}
|
||||
|
||||
// Set inside the transaction by a cap-hit coalesce; flushed AFTER commit
|
||||
// so audit filesystem I/O never runs while holding the advisory lock.
|
||||
let coalesceAudit: CoalesceAuditEvent | null = null;
|
||||
@@ -251,6 +308,75 @@ export class MinionQueue {
|
||||
}
|
||||
}
|
||||
|
||||
// 1a. Param-coalescing (admission): an identical parentless submit —
|
||||
// same (name, queue, payload hash, incl. __owner_client_id so owner
|
||||
// lanes never cross) — returns the newest matching WAITING row instead
|
||||
// of inserting a duplicate. Honest-dispatch contract holds (coalesced:
|
||||
// true), and unlike the cap-hit coalesce below, returning this row to a
|
||||
// result-consumer is semantically exact: identical params ⇒ identical
|
||||
// result. Waiting-only by design (a RUNNING identical job does not
|
||||
// suppress a re-run; maxPending exists for single-flight callers).
|
||||
// Age-bounded to ttl/2: coalescing onto a nearly-TTL-expired row would
|
||||
// silently kill the fresh intent an hour later (round-2 V7).
|
||||
if (coalesceActive && paramHash) {
|
||||
const admissionQueue = opts?.queue ?? 'default';
|
||||
await tx.executeRaw(
|
||||
`SELECT pg_advisory_xact_lock(hashtext('minion_admission:' || $1 || ':' || $2 || ':' || $3))`,
|
||||
[jobName, admissionQueue, paramHash]
|
||||
);
|
||||
const ttlHours = policy.ttlWaitingHours;
|
||||
// updated_at, matching the TTL sweep's key: a requeued row has a
|
||||
// fresh TTL window and is a legitimate coalesce target again.
|
||||
const ageCond = ttlHours != null
|
||||
? `AND updated_at > now() - ($4 * interval '1 hour')`
|
||||
: '';
|
||||
const matchParams: unknown[] = [jobName, admissionQueue, paramHash];
|
||||
if (ttlHours != null) matchParams.push(ttlHours / 2);
|
||||
const match = await tx.executeRaw<Record<string, unknown>>(
|
||||
`SELECT * FROM minion_jobs
|
||||
WHERE name = $1 AND queue = $2 AND status = 'waiting'
|
||||
AND parent_job_id IS NULL
|
||||
AND data->>'__param_hash' = $3
|
||||
${ageCond}
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1`,
|
||||
matchParams
|
||||
);
|
||||
if (match.length > 0) {
|
||||
return coalesceReturn(match[0], {
|
||||
queue: admissionQueue,
|
||||
name: jobName,
|
||||
param_hash: paramHash,
|
||||
}, ev => { coalesceAudit = ev; });
|
||||
}
|
||||
}
|
||||
|
||||
// 1a2. Name-global waiting quota (admission; config-only, no shipped
|
||||
// default — user decision D2C). Counts the name across ALL queues:
|
||||
// fanout producers use per-run private queues (dream-inline-*), so a
|
||||
// queue-scoped count would reset per run and never bind. REJECTION,
|
||||
// not coalesce — quota-coalescing would hand result-consumers an
|
||||
// unrelated row. The name-global advisory lock below serializes
|
||||
// check+insert across concurrent submitters (adversarial finding:
|
||||
// without it, N parallel distinct-payload submits each observed
|
||||
// capacity and inserted, so overshoot was bounded only by attacker
|
||||
// concurrency — defeating the DoS backstop). Serialization cost only
|
||||
// applies to names with a quota configured, i.e. the runaway ones.
|
||||
if (policy.quotaMaxWaiting != null) {
|
||||
await tx.executeRaw(
|
||||
`SELECT pg_advisory_xact_lock(hashtext('minion_quota:' || $1))`,
|
||||
[jobName]
|
||||
);
|
||||
const quotaRows = await tx.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs WHERE name = $1 AND status = 'waiting'`,
|
||||
[jobName]
|
||||
);
|
||||
const waitingTotal = parseInt(quotaRows[0]?.count ?? '0', 10);
|
||||
if (waitingTotal >= policy.quotaMaxWaiting) {
|
||||
throw new QueueQuotaExceededError(jobName, waitingTotal, policy.quotaMaxWaiting);
|
||||
}
|
||||
}
|
||||
|
||||
// 1b. Submission-time backpressure for high-frequency named jobs.
|
||||
// Two guards share the advisory-lock machinery but differ in what they
|
||||
// count and how they scope:
|
||||
@@ -578,10 +704,40 @@ export class MinionQueue {
|
||||
* Returns the *root* (the job matching id), not an arbitrary descendant.
|
||||
*/
|
||||
async cancelJob(id: number): Promise<MinionJob | null> {
|
||||
const cancelled = await this.cancelJobs([id]);
|
||||
const root = cancelled.find(j => j.id === id);
|
||||
return root ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch variant of cancelJob: cancels every root id AND its descendants in
|
||||
* ONE transaction, with the full bookkeeping the single-id path carries
|
||||
* (child_done inbox messages + aggregator-parent resolution). Callers that
|
||||
* cancel in bulk (the waiting-TTL sweep) MUST use this — a raw set-based
|
||||
* UPDATE would skip that bookkeeping and wedge parents in waiting-children
|
||||
* forever (the exact wedge class this wave fights).
|
||||
*
|
||||
* `opts.reason` is written to error_text (COALESCE-preserved when a row
|
||||
* already carries one). The single-id UPDATE never wrote error_text, so
|
||||
* every surface keyed on a reason prefix (jobs stats, doctor) would
|
||||
* silently report zero without this parameter.
|
||||
*/
|
||||
async cancelJobs(ids: number[], opts?: { reason?: string; rootStatuses?: MinionJobStatus[] }): Promise<MinionJob[]> {
|
||||
if (ids.length === 0) return [];
|
||||
// opts.rootStatuses re-checks each ROOT id's status ATOMICALLY inside the
|
||||
// cancel UPDATE's CTE seed. The waiting-TTL sweep passes ['waiting'] to
|
||||
// close its SELECT→cancel race: claim() and the sweep both target the
|
||||
// oldest waiting rows, so without this a job claimed between the sweep's
|
||||
// SELECT and this UPDATE would be cancelled while ACTIVE (lock_token
|
||||
// NULLed under the running handler). Operator cancels omit it — killing
|
||||
// an active job is exactly what `jobs cancel` means.
|
||||
const rootStatuses = opts?.rootStatuses ?? null;
|
||||
return this.engine.transaction(async (tx) => {
|
||||
const rows = await tx.executeRaw<Record<string, unknown>>(
|
||||
`WITH RECURSIVE descendants AS (
|
||||
SELECT id, 0 AS d FROM minion_jobs WHERE id = $1
|
||||
SELECT id, 0 AS d FROM minion_jobs
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ($3::text[] IS NULL OR status = ANY($3::text[]))
|
||||
UNION ALL
|
||||
SELECT m.id, descendants.d + 1
|
||||
FROM minion_jobs m
|
||||
@@ -592,14 +748,20 @@ export class MinionQueue {
|
||||
status = 'cancelled',
|
||||
lock_token = NULL,
|
||||
lock_until = NULL,
|
||||
-- Reason stamps ROOT ids only: a descendant is bookkeeping-cancelled
|
||||
-- because its parent went away, not because IT hit the caller's
|
||||
-- reason (e.g. a waiting-TTL child would otherwise carry a factually
|
||||
-- false 'waited > Nh' text AND inflate the LIKE-prefix stats the
|
||||
-- alerting surfaces count).
|
||||
error_text = CASE WHEN id = ANY($1::int[]) THEN COALESCE($2, error_text) ELSE error_text END,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id IN (SELECT id FROM descendants)
|
||||
AND status IN ('waiting','active','delayed','waiting-children','paused')
|
||||
RETURNING *`,
|
||||
[id]
|
||||
[ids, opts?.reason ?? null, rootStatuses]
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
// v0.15: emit child_done(outcome='cancelled') for every cancelled row
|
||||
// that had a parent. Without this, an aggregator waiting for N
|
||||
@@ -652,11 +814,68 @@ export class MinionQueue {
|
||||
);
|
||||
}
|
||||
|
||||
const root = rows.find(r => (r.id as number) === id);
|
||||
return root ? rowToMinionJob(root) : null;
|
||||
return rows.map(rowToMinionJob);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Waiting-TTL sweep (admission control, run from the worker's maintenance
|
||||
* interval): cancel jobs still WAITING past their per-name TTL, via
|
||||
* cancelJobs() so descendants cancel and aggregator parents resolve.
|
||||
*
|
||||
* `maxPerTick` bounds one tick's work (default 500) so the first
|
||||
* post-upgrade tick against a large backlog can't stall the maintenance
|
||||
* loop — the backlog drains over a few ticks. Oldest-first so FIFO
|
||||
* fairness of what REMAINS is preserved.
|
||||
*
|
||||
* NOTE (warn-before-act, user requirement D1A): the worker gates this
|
||||
* sweep behind the one-time `minions.ttl_notice_shown` flag — tick 1
|
||||
* counts + warns, sweeping starts on tick 2. The gate lives in worker.ts
|
||||
* (the channel where the notice prints); this method just sweeps.
|
||||
*/
|
||||
async handleWaitingTTL(opts?: { maxPerTick?: number }): Promise<{ cancelled: number; by_name: Record<string, number> }> {
|
||||
const maxPerTick = Math.max(1, Math.floor(opts?.maxPerTick ?? 500));
|
||||
const ttlNames = await resolveTtlNames(this.engine);
|
||||
const by_name: Record<string, number> = {};
|
||||
let cancelled = 0;
|
||||
for (const [name, hours] of ttlNames) {
|
||||
if (cancelled >= maxPerTick) break;
|
||||
const budget = maxPerTick - cancelled;
|
||||
// Keyed on updated_at (last state transition), NOT created_at: every
|
||||
// path that RETURNS a row to 'waiting' — handleStalled's requeue (sweep
|
||||
// #1 of the SAME maintenance tick), retryJob, promoteDelayed, the
|
||||
// parent-unblock flips — bumps updated_at but not created_at. Keying on
|
||||
// created_at would cancel a job the same tick just requeued it, with a
|
||||
// "waited > Nh" reason that is factually false. For rows that sat
|
||||
// untouched in 'waiting' the two timestamps are equivalent, so the
|
||||
// backlog-drain semantics are unchanged.
|
||||
const stale = await this.engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM minion_jobs
|
||||
WHERE name = $1 AND status = 'waiting'
|
||||
AND updated_at < now() - ($2 * interval '1 hour')
|
||||
ORDER BY updated_at ASC
|
||||
LIMIT $3`,
|
||||
[name, hours, budget]
|
||||
);
|
||||
if (stale.length === 0) continue;
|
||||
const reason =
|
||||
`${TTL_REASON_PREFIX}: waited > ${hours}h in queue ` +
|
||||
`(minions.ttl_waiting_hours.${name}; set 0 to disable)`;
|
||||
// rootStatuses:['waiting'] re-checks atomically inside the cancel — a
|
||||
// job CLAIMED between the SELECT above and this UPDATE must not be
|
||||
// cancelled mid-run (both the claimer and this sweep target the oldest
|
||||
// waiting rows, so the race is systematic, not incidental).
|
||||
const swept = await this.cancelJobs(stale.map(r => r.id), { reason, rootStatuses: ['waiting'] });
|
||||
// Count only the requested roots — descendants of a swept parent are
|
||||
// bookkeeping, not TTL victims of their own.
|
||||
const rootIds = new Set(stale.map(r => r.id));
|
||||
const rootCount = swept.filter(j => rootIds.has(j.id)).length;
|
||||
by_name[name] = (by_name[name] ?? 0) + rootCount;
|
||||
cancelled += rootCount;
|
||||
}
|
||||
return { cancelled, by_name };
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-queue a failed or dead job for retry.
|
||||
*
|
||||
@@ -723,7 +942,19 @@ export class MinionQueue {
|
||||
/** Get job statistics. */
|
||||
async getStats(opts?: { since?: Date; queue?: string }): Promise<{
|
||||
by_status: Record<string, number>;
|
||||
by_type: Array<{ name: string; total: number; completed: number; failed: number; dead: number; avg_duration_ms: number | null }>;
|
||||
/**
|
||||
* Per-type window stats. `total` counts rows CREATED in the window
|
||||
* (intake). The `drained_*` fields count rows that reached a terminal
|
||||
* status IN the window (`finished_at >= since`) regardless of when they
|
||||
* were created — the true outflow. They are split by terminal status
|
||||
* because a naive combined "drain" number self-inflates on TTL/manual
|
||||
* cancellations while zero useful work happens; divergence alerting
|
||||
* compares intake against drained_completed. `waiting_now` and
|
||||
* `oldest_waiting_minutes` are point-in-time (not windowed).
|
||||
*/
|
||||
by_type: Array<{ name: string; total: number; completed: number; failed: number; dead: number; avg_duration_ms: number | null;
|
||||
drained_completed: number; drained_failed: number; drained_dead: number; drained_cancelled: number;
|
||||
waiting_now: number; oldest_waiting_minutes: number | null }>;
|
||||
queue_health: { waiting: number; active: number; stalled: number };
|
||||
/**
|
||||
* issue #1801 — QUEUE-SCOPED wedge signature for the `jobs stats` WEDGED
|
||||
@@ -763,14 +994,63 @@ export class MinionQueue {
|
||||
GROUP BY name ORDER BY total DESC`,
|
||||
[since.toISOString()]
|
||||
);
|
||||
const by_type = typeRows.map(r => ({
|
||||
name: r.name as string,
|
||||
total: parseInt(r.total as string, 10),
|
||||
completed: parseInt(r.completed as string, 10),
|
||||
failed: parseInt(r.failed as string, 10),
|
||||
dead: parseInt(r.dead as string, 10),
|
||||
avg_duration_ms: r.avg_duration_ms != null ? Math.round(r.avg_duration_ms as number) : null,
|
||||
}));
|
||||
// True per-type outflow: rows that reached a terminal status IN the
|
||||
// window, keyed on finished_at (a row created last week and finished
|
||||
// today drained today). Split by status — cancellations (incl. the
|
||||
// waiting-TTL sweep) are outflow but not useful work.
|
||||
const drainRows = await this.engine.executeRaw<Record<string, unknown>>(
|
||||
`SELECT name,
|
||||
count(*) FILTER (WHERE status = 'completed')::text AS d_completed,
|
||||
count(*) FILTER (WHERE status = 'failed')::text AS d_failed,
|
||||
count(*) FILTER (WHERE status = 'dead')::text AS d_dead,
|
||||
count(*) FILTER (WHERE status = 'cancelled')::text AS d_cancelled
|
||||
FROM minion_jobs
|
||||
WHERE finished_at IS NOT NULL AND finished_at >= $1
|
||||
AND status IN ('completed','failed','dead','cancelled')
|
||||
GROUP BY name`,
|
||||
[since.toISOString()]
|
||||
);
|
||||
const drainByName = new Map(drainRows.map(r => [r.name as string, r]));
|
||||
|
||||
// Point-in-time per-type waiting depth + oldest wait age.
|
||||
const depthRows = await this.engine.executeRaw<Record<string, unknown>>(
|
||||
`SELECT name,
|
||||
count(*)::text AS waiting_now,
|
||||
EXTRACT(EPOCH FROM (now() - min(created_at)))::text AS oldest_waiting_seconds
|
||||
FROM minion_jobs WHERE status = 'waiting'
|
||||
GROUP BY name`,
|
||||
[]
|
||||
);
|
||||
const depthByName = new Map(depthRows.map(r => [r.name as string, r]));
|
||||
|
||||
// Union of names so a type with waiting rows but zero window intake (or
|
||||
// vice versa) still gets a row — divergence alerting needs both sides.
|
||||
const typeNames = new Set<string>([
|
||||
...typeRows.map(r => r.name as string),
|
||||
...drainRows.map(r => r.name as string),
|
||||
...depthRows.map(r => r.name as string),
|
||||
]);
|
||||
const typeByName = new Map(typeRows.map(r => [r.name as string, r]));
|
||||
const by_type = [...typeNames].map(name => {
|
||||
const r = typeByName.get(name);
|
||||
const d = drainByName.get(name);
|
||||
const w = depthByName.get(name);
|
||||
const oldest = w?.oldest_waiting_seconds != null ? Number(w.oldest_waiting_seconds) : null;
|
||||
return {
|
||||
name,
|
||||
total: r ? parseInt(r.total as string, 10) : 0,
|
||||
completed: r ? parseInt(r.completed as string, 10) : 0,
|
||||
failed: r ? parseInt(r.failed as string, 10) : 0,
|
||||
dead: r ? parseInt(r.dead as string, 10) : 0,
|
||||
avg_duration_ms: r?.avg_duration_ms != null ? Math.round(r.avg_duration_ms as number) : null,
|
||||
drained_completed: d ? parseInt(d.d_completed as string, 10) : 0,
|
||||
drained_failed: d ? parseInt(d.d_failed as string, 10) : 0,
|
||||
drained_dead: d ? parseInt(d.d_dead as string, 10) : 0,
|
||||
drained_cancelled: d ? parseInt(d.d_cancelled as string, 10) : 0,
|
||||
waiting_now: w ? parseInt(w.waiting_now as string, 10) : 0,
|
||||
oldest_waiting_minutes: oldest != null && Number.isFinite(oldest) ? Math.round(oldest / 60) : null,
|
||||
};
|
||||
}).sort((a, b) => b.total - a.total);
|
||||
|
||||
// Queue health: stalled = active with expired lock
|
||||
const stalledRows = await this.engine.executeRaw<{ count: string }>(
|
||||
@@ -1822,3 +2102,25 @@ export class MinionQueue {
|
||||
return rows.length > 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1801 / CLI→MCP gap-closure wave (CEO-F7) — derive the wedged-queue
|
||||
* boolean from getStats().wedge, shared by `gbrain jobs stats` and the
|
||||
* get_job_stats op so the two surfaces can never disagree. A queue is wedged
|
||||
* when a worker is alive but claiming nothing while work waits:
|
||||
* zero live-lock active rows, waiting > 0, and the last completion is older
|
||||
* than the threshold (or absent). Threshold matches the doctor wedged_queue
|
||||
* check: GBRAIN_WEDGED_QUEUE_WARN_MINUTES (server-side env), default 15.
|
||||
*/
|
||||
export function deriveWedgeSignal(wedge: {
|
||||
active_healthy: number;
|
||||
waiting: number;
|
||||
minutes_since_completion: number | null;
|
||||
}): { wedged: boolean; wedge_threshold_minutes: number } {
|
||||
const raw = parseInt(process.env.GBRAIN_WEDGED_QUEUE_WARN_MINUTES ?? '', 10);
|
||||
const wedge_threshold_minutes = Number.isFinite(raw) && raw > 0 ? raw : 15;
|
||||
const mins = wedge.minutes_since_completion;
|
||||
const wedged = wedge.active_healthy === 0 && wedge.waiting > 0
|
||||
&& (mins === null || mins > wedge_threshold_minutes);
|
||||
return { wedged, wedge_threshold_minutes };
|
||||
}
|
||||
|
||||
@@ -163,6 +163,17 @@ export interface MinionJobInput {
|
||||
* as a public submit flag yet — semantics exclude delayed/paused/
|
||||
* waiting-children rows deliberately. */
|
||||
maxPending?: number;
|
||||
/**
|
||||
* Admission param-coalescing override. When unset, the per-name default
|
||||
* (admission.ts PARAM_COALESCE_DEFAULT, config-overridable via
|
||||
* minions.coalesce_params.<name>) applies — on for 'subagent'. When
|
||||
* active, a parentless submit whose payload hash (sha256 of
|
||||
* stable-stringified data, __owner_client_id INCLUDED so owner lanes never
|
||||
* cross) matches a WAITING row for the same (name, queue) returns that row
|
||||
* with `coalesced: true` instead of inserting a duplicate. Parented
|
||||
* submits never coalesce regardless of this flag.
|
||||
*/
|
||||
coalesce_params?: boolean;
|
||||
|
||||
// v12: scheduler polish
|
||||
/**
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
ABORT_REASON_LOCK_LOST,
|
||||
} from './types.ts';
|
||||
import { MinionQueue } from './queue.ts';
|
||||
import { runWaitingTtlTick, ttlNoticeGraceMs } from './admission.ts';
|
||||
import { calculateBackoff } from './backoff.ts';
|
||||
import { RateLeaseUnavailableError } from './handlers/subagent.ts';
|
||||
import { logLeasePressure } from './lease-pressure-audit.ts';
|
||||
@@ -438,6 +439,33 @@ export class MinionWorker extends EventEmitter {
|
||||
console.error('Wall-clock timeout detection error:', e instanceof Error ? e.message : String(e));
|
||||
await recoverConnection('handleWallClockTimeouts', e);
|
||||
}
|
||||
// 4th sweep: waiting-TTL (admission control). Warn-before-act (user
|
||||
// requirement D1A) lives in runWaitingTtlTick (admission.ts): the first
|
||||
// tick counts + stamps the notice timestamp, sweeping starts only after
|
||||
// the grace window elapses. The gate is engine-state (not process
|
||||
// state) because the worker restarts via self-upgrade/systemd without
|
||||
// ever running the CLI's runPostUpgrade banner; the notice must precede
|
||||
// the cancellation on EVERY channel, and the worker log is the daemon
|
||||
// channel.
|
||||
try {
|
||||
const tick = await runWaitingTtlTick(this.engine, this.queue);
|
||||
if (tick.phase === 'notice') {
|
||||
console.log(
|
||||
`⚠ Waiting-TTL is now active: ${tick.affected ?? 0} queued job(s) currently exceed their TTL and ` +
|
||||
`will be cancelled after a ${Math.round(ttlNoticeGraceMs() / 60_000)}min grace window. ` +
|
||||
`Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>.`,
|
||||
);
|
||||
} else if (tick.phase === 'swept' && tick.cancelled > 0) {
|
||||
const breakdown = Object.entries(tick.by_name).map(([n, c]) => `${n}: ${c}`).join(', ');
|
||||
console.log(
|
||||
`Waiting-TTL: cancelled ${tick.cancelled} job(s) that waited past their TTL (${breakdown}). ` +
|
||||
`Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>. See 'gbrain jobs stats'.`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Waiting-TTL sweep error:', e instanceof Error ? e.message : String(e));
|
||||
await recoverConnection('handleWaitingTTL', e);
|
||||
}
|
||||
}, this.opts.stalledInterval);
|
||||
|
||||
// Periodic RSS watchdog — closes the production-freeze regression where
|
||||
|
||||
@@ -238,3 +238,19 @@ export const SKILL_CLIENT_GUIDANCE = {
|
||||
"if the user hasn't clearly asked for a write.",
|
||||
],
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* CLI→MCP gap-closure wave — the capture op (D2A). Pinned here because it
|
||||
* rewrites the routing guidance three docs used to carry as the
|
||||
* "unknown tool: capture → use put_page" FAQ: agents must learn the split
|
||||
* (capture = quick notes with auto-slug + dedupe; put_page = full control)
|
||||
* from this description alone. Phrase-pinned by
|
||||
* test/operations-descriptions.test.ts.
|
||||
*/
|
||||
export const CAPTURE_DESCRIPTION =
|
||||
'Capture a quick note into the brain — the "just remember this" write. Auto-derives a ' +
|
||||
'stable inbox/ slug from the content date + hash (recapturing identical text is ' +
|
||||
'idempotent), merges frontmatter, refuses binary/empty payloads, then delegates to ' +
|
||||
'put_page (inheriting its fences and provenance stamping). Prefer capture for quick ' +
|
||||
'notes and put_page when you need to control the slug, type, or an existing page\'s ' +
|
||||
'content. For structured facts about entities, prefer remember.';
|
||||
|
||||
+57
-3
@@ -28,10 +28,24 @@ const get_stats: Operation = {
|
||||
|
||||
const get_health: Operation = {
|
||||
name: 'get_health',
|
||||
description: 'Brain health dashboard (embed coverage, stale pages, orphans)',
|
||||
description: 'Brain health dashboard (embed coverage, stale pages, orphans). Includes a `migrations {pending, partial, wedged, skipped_future}` block from the host migration ledger so remote agents can detect wedged/outstanding host migrations without shelling into the brain host.',
|
||||
params: {},
|
||||
handler: async (ctx) => {
|
||||
return ctx.engine.getHealth();
|
||||
const health = await ctx.engine.getHealth();
|
||||
// TODOS:4063 — composed at the OP layer (not BrainEngine.getHealth):
|
||||
// the ledger is a filesystem JSONL, engine-agnostic; growing the engine
|
||||
// interface would force both engines to duplicate a file read.
|
||||
// Best-effort like the doctor's ledger read: a corrupt/unreadable ledger
|
||||
// degrades the field, never the health call.
|
||||
let migrations: unknown;
|
||||
try {
|
||||
const { migrationLedgerSummary } = await import('../migration-ledger.ts');
|
||||
const { VERSION } = await import('../../version.ts');
|
||||
migrations = migrationLedgerSummary(VERSION);
|
||||
} catch {
|
||||
migrations = { error: 'ledger_unreadable' };
|
||||
}
|
||||
return { ...health, migrations };
|
||||
},
|
||||
scope: 'admin',
|
||||
cliHints: { name: 'health' },
|
||||
@@ -164,8 +178,48 @@ const revert_version: Operation = {
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* CLI→MCP gap-closure wave — read-only view of the content-quality gate
|
||||
* (issue #1699). User story: an operator/agent reviewing what the gate hid or
|
||||
* flagged, remotely or via a thin-client CLI. Admin scope: enumerates
|
||||
* deliberately-hidden page slugs (the run_doctor posture). `quarantine scan`
|
||||
* (bulk re-import + re-embed) and `quarantine clear` (the trust decision —
|
||||
* same class as extraction_review) stay CLI-only.
|
||||
*/
|
||||
const quarantine_list: Operation = {
|
||||
name: 'quarantine_list',
|
||||
description:
|
||||
'List quarantined (hidden) and optionally content-flagged pages by scanning page ' +
|
||||
'frontmatter, newest-updated first. When truncated is true, count is a LOWER BOUND — ' +
|
||||
'raise max_scan/limit or run the quarantine list command on the brain host for the full ' +
|
||||
'set. Clearing a marker is a local-only trust decision (CLI).',
|
||||
params: {
|
||||
include_flagged: { type: 'boolean', required: false, description: 'Also list content_flag pages (searchable-but-warned). Default false.' },
|
||||
limit: { type: 'number', required: false, description: 'Max rows returned (default 200, cap 1000).' },
|
||||
max_scan: { type: 'number', required: false, description: 'Max pages scanned (default 20000, cap 100000).' },
|
||||
},
|
||||
scope: 'admin',
|
||||
area: 'admin',
|
||||
handler: async (ctx, p) => {
|
||||
const {
|
||||
collectQuarantineRows,
|
||||
QUARANTINE_LIST_DEFAULT_LIMIT, QUARANTINE_LIST_MAX_LIMIT,
|
||||
QUARANTINE_SCAN_DEFAULT, QUARANTINE_SCAN_MAX,
|
||||
} = await import('../../commands/quarantine.ts');
|
||||
const rawLimit = typeof p.limit === 'number' && Number.isFinite(p.limit) ? p.limit : QUARANTINE_LIST_DEFAULT_LIMIT;
|
||||
const rawScan = typeof p.max_scan === 'number' && Number.isFinite(p.max_scan) ? p.max_scan : QUARANTINE_SCAN_DEFAULT;
|
||||
const { rows, scanned, truncated } = await collectQuarantineRows(ctx.engine, {
|
||||
includeFlagged: p.include_flagged === true,
|
||||
limit: Math.max(1, Math.min(QUARANTINE_LIST_MAX_LIMIT, rawLimit)),
|
||||
maxScan: Math.max(1, Math.min(QUARANTINE_SCAN_MAX, rawScan)),
|
||||
...sourceScopeOpts(ctx),
|
||||
});
|
||||
return { schema_version: 1, count: rows.length, truncated, scanned, rows };
|
||||
},
|
||||
};
|
||||
|
||||
// Ops in EXACTLY the canonical `operations` array order.
|
||||
export const adminOperations: Operation[] = [
|
||||
get_stats, get_health, run_doctor, get_versions, revert_version,
|
||||
get_brain_identity,
|
||||
get_brain_identity, quarantine_list,
|
||||
];
|
||||
|
||||
@@ -287,6 +287,14 @@ export const CLIENT_FENCED_WRITE_OPS: ReadonlySet<string> = new Set([
|
||||
// would break the original feature for clients that legitimately hold both
|
||||
// a binding and `agent` scope.
|
||||
'submit_agent',
|
||||
// CLI→MCP gap-closure wave: capture delegates to put_page with the same ctx
|
||||
// (inheriting its enforceClientSlugFence) and [EV7] defaults its slug UNDER
|
||||
// the caller's first bound prefix — the zero-config path exists for exactly
|
||||
// the bound-agent audience. The takes write verbs each call
|
||||
// enforceClientSlugFence themselves (their markdown mirror writes the
|
||||
// page file under the slug), the same guarantee as add_tag/add_timeline_entry.
|
||||
'capture',
|
||||
'takes_add', 'takes_update', 'takes_resolve', 'takes_supersede',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -78,6 +78,30 @@ export class OperationError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI→MCP gap-closure wave (ENG-E2) — shared relation-missing guard for the
|
||||
* telemetry ops (get_job_stats, cache_stats, search_stats, search_tune).
|
||||
* Older brains that predate a feature's tables raw-error with Postgres 42P01
|
||||
* ("relation … does not exist") / PGLite's equivalent; every telemetry op
|
||||
* must degrade to the SAME actionable 'unavailable' envelope instead — one
|
||||
* helper, never four inline copies.
|
||||
*/
|
||||
export async function withRelationGuard<T>(fn: () => Promise<T>, what: string): Promise<T> {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (/relation .* does not exist|no such table/i.test(msg)) {
|
||||
throw new OperationError(
|
||||
'unavailable',
|
||||
`${what} is unavailable on this brain: a required table is missing.`,
|
||||
'Run gbrain apply-migrations on the brain host, then retry.',
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MEMORY_VERBS v1 error constructor. Every verb error carries a populated
|
||||
* `suggestion` (problem + cause + fix — agents read it and self-correct;
|
||||
|
||||
@@ -181,6 +181,9 @@ const extraction_review: Operation = {
|
||||
if (ctx.dryRun) return { dry_run: true, action: `extraction_review:${action}`, slugs };
|
||||
const results: Array<{ slug: string; status: string }> = [];
|
||||
for (const slug of slugs) {
|
||||
// First-match read is safe here: BOTH writes below key on the RETURNED
|
||||
// row's page.source_id, so read and write can never target different
|
||||
// rows. gbrain-allow-unscoped-getpage: write follows the returned row
|
||||
const page = await ctx.engine.getPage(slug, ctx.sourceId ? { sourceId: ctx.sourceId } : undefined);
|
||||
if (!page) {
|
||||
results.push({ slug, status: 'not_found' });
|
||||
|
||||
@@ -155,7 +155,12 @@ const find_experts: Operation = {
|
||||
...sourceScopeOpts(ctx),
|
||||
});
|
||||
},
|
||||
cliHints: { name: 'whoknows', positional: ['topic'] },
|
||||
// hidden: 'whoknows' is in CLI_ONLY (src/cli.ts) — runWhoknows owns the CLI
|
||||
// surface (ranked table + per-factor explain + thin-client routing) and was
|
||||
// unreachable while this non-hidden hint dispatched the generic op formatter
|
||||
// (the #2035 calibration bug class, resolved the #3502 way: wire the richer
|
||||
// handler, hide the hint).
|
||||
cliHints: { name: 'whoknows', positional: ['topic'], hidden: true },
|
||||
};
|
||||
|
||||
// v0.32.6: contradiction probe MCP surface (M3)
|
||||
|
||||
+63
-7
@@ -328,12 +328,24 @@ const submit_agent: Operation = {
|
||||
);
|
||||
}
|
||||
if (delegatedSource) jobData.source_id = delegatedSource;
|
||||
const job = await queue.add(
|
||||
'subagent',
|
||||
jobData,
|
||||
{ queue: (p.queue as string) || 'default' },
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
let job;
|
||||
try {
|
||||
job = await queue.add(
|
||||
'subagent',
|
||||
jobData,
|
||||
{ queue: (p.queue as string) || 'default' },
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only):
|
||||
// surface as a structured retryable error, not an opaque internal one.
|
||||
// The quota message already omits live cross-tenant queue depth.
|
||||
const { isQueueQuotaExceededError } = await import('../minions/admission.ts');
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
throw new OperationError('rate_limited', e.message, 'Retry after the queue drains, or ask the operator to raise the quota.');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Audit trail (D4) — best-effort JSONL.
|
||||
try {
|
||||
@@ -360,6 +372,12 @@ const submit_agent: Operation = {
|
||||
id: job.id,
|
||||
name: 'subagent',
|
||||
client_id: clientId,
|
||||
// Honest-dispatch: true when this submit was param-coalesced onto an
|
||||
// existing WAITING job with identical params (same owner lane) instead
|
||||
// of enqueuing a new one. Clients wanting N independent runs of one
|
||||
// prompt should vary the params (adversarial-review finding — the flag
|
||||
// makes the suppression detectable rather than silent).
|
||||
...(job.coalesced === true ? { coalesced: true } : {}),
|
||||
queue_state: await probeQueueStateSafe(ctx, job.queue, ['subagent']),
|
||||
};
|
||||
},
|
||||
@@ -628,9 +646,47 @@ const send_job_message: Operation = {
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* CLI→MCP gap-closure wave — `gbrain jobs stats` was the one jobs verb with
|
||||
* no MCP equivalent (skills/minion-orchestrator documented the gap). Admin
|
||||
* scope for jobs-family consistency (every op above is admin; HTTP callers
|
||||
* need an admin-scope token). User story: an orchestrating agent checking
|
||||
* queue health / catching the silent-halt wedge without shelling out.
|
||||
*/
|
||||
const get_job_stats: Operation = {
|
||||
name: 'get_job_stats',
|
||||
description:
|
||||
'Job queue statistics. PER-BLOCK scoping: by_status and queue_health are GLOBAL ' +
|
||||
'(unfiltered); by_type is windowed by since_hours; only the wedge block is scoped to ' +
|
||||
'the queue param. wedged: true is the silent-halt signal (a worker is alive but claiming ' +
|
||||
'nothing while work waits) — suggest restarting the jobs supervisor on the brain host. ' +
|
||||
'Host-process diagnostics (renice, backpressure hints) stay on the gbrain jobs stats CLI.',
|
||||
params: {
|
||||
queue: { type: 'string', required: false, description: "Queue for the wedge signature (default 'default'). The other blocks stay global/windowed." },
|
||||
since_hours: { type: 'number', required: false, description: 'Window for the by_type rollup in hours (default 24, clamped 1..720).' },
|
||||
},
|
||||
scope: 'admin',
|
||||
area: 'jobs',
|
||||
handler: async (ctx, p) => {
|
||||
const { withRelationGuard } = await import('./contract.ts');
|
||||
return withRelationGuard(async () => {
|
||||
const { MinionQueue, deriveWedgeSignal } = await import('../minions/queue.ts');
|
||||
const queue = new MinionQueue(ctx.engine);
|
||||
const rawHours = typeof p.since_hours === 'number' && Number.isFinite(p.since_hours) ? p.since_hours : 24;
|
||||
const hours = Math.max(1, Math.min(720, rawHours));
|
||||
const stats = await queue.getStats({
|
||||
since: new Date(Date.now() - hours * 3_600_000),
|
||||
queue: typeof p.queue === 'string' && p.queue.length > 0 ? p.queue : 'default',
|
||||
});
|
||||
const { wedged, wedge_threshold_minutes } = deriveWedgeSignal(stats.wedge);
|
||||
return { schema_version: 1, window_hours: hours, ...stats, wedged, wedge_threshold_minutes };
|
||||
}, 'Job queue statistics (minions schema)');
|
||||
},
|
||||
};
|
||||
|
||||
// Ops in EXACTLY the canonical `operations` array order.
|
||||
export const jobsOperations: Operation[] = [
|
||||
submit_job, get_job, list_jobs, cancel_job, retry_job, get_job_progress,
|
||||
pause_job, resume_job, replay_job, send_job_message,
|
||||
submit_agent, get_agent_job,
|
||||
submit_agent, get_agent_job, get_job_stats,
|
||||
];
|
||||
|
||||
+91
-3
@@ -19,7 +19,7 @@ import type { WriterLintPayload } from '../output/post-write.ts';
|
||||
import { stripFactsFence } from '../facts-fence.ts';
|
||||
import { getContentFlag } from '../quarantine.ts';
|
||||
import { bumpLastRetrievedAt } from '../last-retrieved.ts';
|
||||
import { LIST_PAGES_DESCRIPTION } from '../operations-descriptions.ts';
|
||||
import { LIST_PAGES_DESCRIPTION, CAPTURE_DESCRIPTION } from '../operations-descriptions.ts';
|
||||
import { OperationError } from './contract.ts';
|
||||
import type { Operation } from './contract.ts';
|
||||
import {
|
||||
@@ -27,6 +27,8 @@ import {
|
||||
slugOutsideCallerFence,
|
||||
enforceClientSlugFence,
|
||||
federatedSearchScope,
|
||||
normalizeSlugPrefix,
|
||||
validatePageSlug,
|
||||
} from './context.ts';
|
||||
|
||||
// --- Page CRUD ---
|
||||
@@ -930,9 +932,95 @@ const list_pages: Operation = {
|
||||
|
||||
|
||||
// Ops in EXACTLY the order they appear in the canonical `operations` array
|
||||
/**
|
||||
* CLI→MCP gap-closure wave — `capture` over MCP (D2A). The documented "just
|
||||
* get this into my brain" entrypoint: three separate docs carried the
|
||||
* "unknown tool: capture → use put_page" FAQ because agents kept reaching for
|
||||
* it. Thin sugar that DELEGATES to the put_page handler with the same ctx
|
||||
* (inheriting every fence: slug fence, dedupe, unknown-type audit,
|
||||
* write-through, remote auto-link skip) after adding what agents had to
|
||||
* hand-roll: a stable content-derived default slug + the frontmatter merge +
|
||||
* the binary/empty guards. Remote provenance stays the CV6 server-stamp
|
||||
* `mcp:put_page` (the write API truthfully IS put_page); the result carries
|
||||
* channel: 'capture' for the receipt. Joins STARTER_OPS as a direct literal
|
||||
* [EV8] so the plugin/starter lanes that retired the FAQ can actually call it.
|
||||
*/
|
||||
const capture: Operation = {
|
||||
name: 'capture',
|
||||
description: CAPTURE_DESCRIPTION,
|
||||
params: {
|
||||
content: { type: 'string', required: true, description: 'Markdown or plain text to capture. File paths are NOT accepted over MCP — read the file yourself and pass its content (the CLI --file lane is local-only).' },
|
||||
slug: { type: 'string', required: false, description: "Target slug. Default: inbox/YYYY-MM-DD-<sha8-of-content> (stable per content — recapturing identical text hits the same slug); type diary/event routes under life/. Fenced clients: the default lands under your first bound prefix." },
|
||||
type: { type: 'string', required: false, description: "Page type for the stamped frontmatter (default 'note')." },
|
||||
},
|
||||
scope: 'write',
|
||||
mutating: true,
|
||||
area: 'pages',
|
||||
// 'capture' is in CLI_ONLY (rich local UX: --file/--stdin/event sugar);
|
||||
// hidden hint per the advisor pattern.
|
||||
cliHints: { name: 'capture', hidden: true },
|
||||
handler: async (ctx, p) => {
|
||||
const {
|
||||
detectBinaryNullByte, normalizeForHash, mergeCaptureFrontmatter,
|
||||
defaultSlug,
|
||||
} = await import('../capture-content.ts');
|
||||
const { computeContentHash } = await import('../ingestion/types.ts');
|
||||
const content = p.content as string;
|
||||
const nulAt = detectBinaryNullByte(Buffer.from(content, 'utf8'));
|
||||
if (nulAt !== -1) {
|
||||
throw new OperationError('invalid_params',
|
||||
`content contains a NUL byte at offset ${nulAt} — binary payloads are refused.`,
|
||||
'Capture takes text/markdown; upload binaries through the files lane on the host.');
|
||||
}
|
||||
const normalized = normalizeForHash(content);
|
||||
if (normalized.length === 0) {
|
||||
throw new OperationError('invalid_params', 'Refusing to capture empty content.');
|
||||
}
|
||||
const type = typeof p.type === 'string' && p.type.length > 0 ? p.type : 'note';
|
||||
let slug = typeof p.slug === 'string' && p.slug.length > 0 ? p.slug : undefined;
|
||||
if (slug) {
|
||||
// Defense-in-depth on the caller-supplied slug (matches the takes ops);
|
||||
// put_page validates again, but reject a malformed slug before we build
|
||||
// provenance frontmatter around it.
|
||||
validatePageSlug(slug);
|
||||
} else {
|
||||
slug = defaultSlug(normalized, new Date(), type);
|
||||
// [EV7] A slug-bound client would 403 on the inbox/ default via the
|
||||
// inherited slug fence — the zero-config path must work for exactly
|
||||
// that audience, so the ENTIRE default slug (type prefix included —
|
||||
// diary/event prefixes are two segments) nests under the FIRST bound
|
||||
// prefix. Normalize a stored `<prefix>/*` glob (submit_agent binding
|
||||
// grammar) to `<prefix>/` first so the nested slug never carries a
|
||||
// literal `*` segment.
|
||||
const bound = ctx.auth?.boundSlugPrefixes;
|
||||
if (bound && bound.length > 0) {
|
||||
const base = normalizeSlugPrefix(bound[0]);
|
||||
const prefix = base.endsWith('/') ? base : `${base}/`;
|
||||
slug = `${prefix}${slug}`;
|
||||
}
|
||||
}
|
||||
// Remote MCP captures record `capture-mcp` provenance; local CLI callers
|
||||
// (ctx.remote === false) keep the neutral 'capture-cli' default.
|
||||
const capturedVia = ctx.remote !== false ? 'capture-mcp' : undefined;
|
||||
const fullContent = mergeCaptureFrontmatter(content, { type, capturedVia });
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'capture', slug };
|
||||
// Delegate with the SAME ctx (the runCapture local-path precedent) —
|
||||
// put_page enforces the slug fence, validates the slug, dedupes, and
|
||||
// server-stamps provenance for remote callers.
|
||||
const result = await put_page.handler(ctx, { slug, content: fullContent }) as Record<string, unknown>;
|
||||
return {
|
||||
...result,
|
||||
slug,
|
||||
channel: 'capture',
|
||||
content_hash: computeContentHash(normalized),
|
||||
dedupe: 'identical normalized content produces the same default slug and hash',
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
// (Page CRUD quartet first, then the v0.26.5 destructive-guard ops:
|
||||
// page-level soft-delete recovery + admin purge).
|
||||
// page-level soft-delete recovery + admin purge, then capture.)
|
||||
export const pagesOperations: Operation[] = [
|
||||
get_page, put_page, delete_page, list_pages,
|
||||
restore_page, purge_deleted_pages,
|
||||
restore_page, purge_deleted_pages, capture,
|
||||
];
|
||||
|
||||
@@ -58,10 +58,14 @@ function firstSentenceOf(description: string): string {
|
||||
* persist level), minus localOnly on network transports (stdio and the
|
||||
* trusted local CLI keep them — D7), minus ops outside the caller's scopes
|
||||
* (agent-callable carve-out per FOV-4), minus bound-client-fenced ops (same
|
||||
* predicate as tools/list, ENG-3), minus publish-gated ops whose gate is off
|
||||
* (stdio and the trusted local CLI bypass gates — the D7 local-surface
|
||||
* posture, matching assertPublishEnabled's remote===false exemption; a
|
||||
* failed gate read hides the gated ops, fail-closed).
|
||||
* predicate as tools/list, ENG-3), minus publish-gated ops whose gate is off.
|
||||
* Two DISTINCT caller axes here: localOnly visibility is the transport-
|
||||
* LOCALITY axis (stdio pipe or trusted local CLI can call localOnly ops, D7),
|
||||
* while publish gates are the owner-CONSENT axis and exempt ONLY
|
||||
* ctx.remote === false (assertPublishEnabled + the advisor inline gate) —
|
||||
* stdio dispatches remote:true, so its catalog subtracts gate-off ops or it
|
||||
* advertises tools that deny at call time. A failed gate read hides the gated
|
||||
* ops, fail-closed.
|
||||
*/
|
||||
async function visibleOpsForCaller(
|
||||
ctx: OperationContext,
|
||||
@@ -73,13 +77,15 @@ async function visibleOpsForCaller(
|
||||
// runs, operations.ts has finished evaluating.
|
||||
const { operations } = await import('../operations.ts');
|
||||
const { filterOpsForSurface } = await import('../../mcp/surface.ts');
|
||||
// Trusted local callers: the stdio pipe, or the local CLI (remote is
|
||||
// strictly false — the fail-closed trust marker). Both CAN call localOnly
|
||||
// and gated ops, so hiding them would make the catalog dishonest.
|
||||
const isLocal = ctx.transport === 'stdio' || ctx.remote === false;
|
||||
// Locality axis: the stdio pipe or the local CLI (remote strictly false)
|
||||
// CAN call localOnly ops, so hiding those would make the catalog dishonest.
|
||||
const canSeeLocalOnly = ctx.transport === 'stdio' || ctx.remote === false;
|
||||
// Consent axis: publish-gate enforcement exempts ONLY remote === false —
|
||||
// stdio is remote:true, so it is gate-subject like every other agent caller.
|
||||
const gateExempt = ctx.remote === false;
|
||||
|
||||
let gateDisabled: ReadonlySet<string> = new Set();
|
||||
if (!isLocal) {
|
||||
if (!gateExempt) {
|
||||
try {
|
||||
const { disabledOpsForPublishGates } = await import('../../mcp/publish-gates.ts');
|
||||
gateDisabled = await disabledOpsForPublishGates(ctx.engine, ctx.config);
|
||||
@@ -96,7 +102,7 @@ async function visibleOpsForCaller(
|
||||
const scopes = ctx.auth?.scopes && ctx.auth.scopes.length > 0 ? ctx.auth.scopes : null;
|
||||
|
||||
return filterOpsForSurface(operations, ceiling).filter(op =>
|
||||
(isLocal || !op.localOnly)
|
||||
(canSeeLocalOnly || !op.localOnly)
|
||||
&& (scopes === null
|
||||
|| hasScope(scopes, op.scope ?? 'read')
|
||||
|| (op.agentCallable === true && hasScope(scopes, 'agent')))
|
||||
|
||||
@@ -51,7 +51,9 @@ const get_recent_salience: Operation = {
|
||||
recency_bias: recencyBias,
|
||||
});
|
||||
},
|
||||
cliHints: { name: 'salience' },
|
||||
// hidden: 'salience' is in CLI_ONLY (src/cli.ts) — runSalience owns the CLI
|
||||
// surface; the non-hidden hint was dead (CLI_ONLY wins at dispatch).
|
||||
cliHints: { name: 'salience', hidden: true },
|
||||
};
|
||||
|
||||
const find_anomalies: Operation = {
|
||||
@@ -79,7 +81,9 @@ const find_anomalies: Operation = {
|
||||
sigma: typeof p.sigma === 'number' ? p.sigma : undefined,
|
||||
});
|
||||
},
|
||||
cliHints: { name: 'anomalies' },
|
||||
// hidden: 'anomalies' is in CLI_ONLY (src/cli.ts) — runAnomalies owns the
|
||||
// CLI surface; the non-hidden hint was dead (CLI_ONLY wins at dispatch).
|
||||
cliHints: { name: 'anomalies', hidden: true },
|
||||
};
|
||||
|
||||
|
||||
|
||||
+113
-1
@@ -370,5 +370,117 @@ const query: Operation = {
|
||||
};
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI→MCP gap-closure wave — search/cache introspection ops. Read-only views
|
||||
// shared with the `gbrain search modes|stats|tune` + `gbrain cache stats` CLI
|
||||
// (the builders live in core/search/). User story for each: a thin-client
|
||||
// user whose CLI routes these subcommands remotely, or an agent asked to
|
||||
// diagnose retrieval quality/cost. Telemetry ops are admin-scoped
|
||||
// (operational counters, the get_status_snapshot posture); search_modes is
|
||||
// read-scoped (resolved knob values only — agents budget their own calls
|
||||
// with it, no usage data).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const search_stats: Operation = {
|
||||
name: 'search_stats',
|
||||
description:
|
||||
'Search observability over a window: cache hit rate, intent/mode mix, budget drops, ' +
|
||||
'rank-1 score drift, graph-signals failure counts. Same payload as the search-stats ' +
|
||||
'dashboard JSON. Coverage caveat: telemetry is best-effort (short-lived CLI calls may ' +
|
||||
'not flush), so zero counts can reflect the coverage gap rather than zero usage.',
|
||||
params: {
|
||||
days: { type: 'number', required: false, description: 'Window in days (default 7, clamped 1..365).' },
|
||||
},
|
||||
scope: 'admin',
|
||||
area: 'search',
|
||||
handler: async (ctx, p) => {
|
||||
const { withRelationGuard } = await import('./contract.ts');
|
||||
return withRelationGuard(async () => {
|
||||
const { readSearchStats, readGraphSignalsStats, telemetryCoverage } = await import('../search/telemetry.ts');
|
||||
const rawDays = typeof p.days === 'number' && Number.isFinite(p.days) ? p.days : 7;
|
||||
const days = Math.max(1, Math.min(365, rawDays));
|
||||
const stats = await readSearchStats(ctx.engine, { days });
|
||||
const graph_signals = await readGraphSignalsStats(ctx.engine, days);
|
||||
return {
|
||||
schema_version: 2,
|
||||
...stats,
|
||||
coverage: telemetryCoverage(),
|
||||
graph_signals,
|
||||
_meta: {
|
||||
metric_glossary: {
|
||||
cache_hit_rate: 'cache_hits / (cache_hits + cache_misses) — fraction of searches that reused a recent answer instead of running fresh',
|
||||
avg_results: 'mean number of result rows returned per search call',
|
||||
avg_tokens: 'mean estimated tokens in the returned chunk text (char/4 heuristic)',
|
||||
total_budget_dropped: 'sum of results dropped because the call exceeded its tokenBudget',
|
||||
graph_signals_enabled: 'whether graph_signals is on for the active mode (or via search.graph_signals override)',
|
||||
graph_signals_failures_count: 'count of fail-open events in the JSONL audit over the window',
|
||||
},
|
||||
},
|
||||
};
|
||||
}, 'Search telemetry');
|
||||
},
|
||||
};
|
||||
|
||||
const search_modes: Operation = {
|
||||
name: 'search_modes',
|
||||
description:
|
||||
'Read-only search-mode dashboard: active mode, per-knob resolved value with attribution ' +
|
||||
'(mode default vs config override), and the three frozen bundles. Never mutates; to ' +
|
||||
'change modes, tell the user to set the search.mode config key on the brain host.',
|
||||
params: {},
|
||||
scope: 'read',
|
||||
area: 'search',
|
||||
handler: async (ctx) => {
|
||||
const { buildModesReport } = await import('../search/modes-report.ts');
|
||||
return buildModesReport(ctx.engine);
|
||||
},
|
||||
};
|
||||
|
||||
const search_tune: Operation = {
|
||||
name: 'search_tune',
|
||||
description:
|
||||
'Read-only tuning recommendations derived from the last 7 days of search telemetry: ' +
|
||||
'what should change, why, and the paste-ready config command per recommendation — relay ' +
|
||||
'them to the user. Applying is CLI-only by design [CDX-21]: this op NEVER mutates config.',
|
||||
params: {},
|
||||
scope: 'admin',
|
||||
area: 'search',
|
||||
handler: async (ctx) => {
|
||||
const { withRelationGuard } = await import('./contract.ts');
|
||||
return withRelationGuard(async () => {
|
||||
const { buildTuneRecommendations } = await import('../search/tune-recommendations.ts');
|
||||
return buildTuneRecommendations(ctx.engine);
|
||||
}, 'Search telemetry');
|
||||
},
|
||||
};
|
||||
|
||||
const cache_stats: Operation = {
|
||||
name: 'cache_stats',
|
||||
description:
|
||||
'Semantic query-cache introspection: resolved knobs (enabled, similarity threshold, TTL) ' +
|
||||
'plus row counts and total hits. Read-only; clearing/pruning the cache stays on the CLI.',
|
||||
params: {},
|
||||
scope: 'admin',
|
||||
area: 'search',
|
||||
handler: async (ctx) => {
|
||||
const { withRelationGuard } = await import('./contract.ts');
|
||||
return withRelationGuard(async () => {
|
||||
const { SemanticQueryCache, loadCacheConfig } = await import('../search/query-cache.ts');
|
||||
const config = await loadCacheConfig(ctx.engine);
|
||||
const cache = new SemanticQueryCache(ctx.engine, config);
|
||||
const stats = await cache.stats();
|
||||
return {
|
||||
schema_version: 1,
|
||||
enabled: config.enabled ?? true,
|
||||
similarity_threshold: config.similarityThreshold,
|
||||
ttl_seconds: config.ttlSeconds,
|
||||
...stats,
|
||||
};
|
||||
}, 'Query-cache statistics');
|
||||
},
|
||||
};
|
||||
|
||||
// Ops in EXACTLY the canonical `operations` array order.
|
||||
export const searchOperations: Operation[] = [search, query];
|
||||
export const searchOperations: Operation[] = [
|
||||
search, query, search_stats, search_modes, search_tune, cache_stats,
|
||||
];
|
||||
|
||||
+358
-5
@@ -5,8 +5,21 @@
|
||||
* in ../operations.ts. Never import from '../operations.ts' here (cycle).
|
||||
*/
|
||||
|
||||
import type { Operation } from './contract.ts';
|
||||
import { sourceScopeOpts, thinkSourceScopeOpts } from './context.ts';
|
||||
import { OperationError, type Operation, type OperationContext } from './contract.ts';
|
||||
import {
|
||||
sourceScopeOpts,
|
||||
thinkSourceScopeOpts,
|
||||
enforceClientSlugFence,
|
||||
validatePageSlug,
|
||||
} from './context.ts';
|
||||
import {
|
||||
addTakeToPage,
|
||||
updateTakeOnPage,
|
||||
supersedeTakeOnPage,
|
||||
resolveTakeOnPage,
|
||||
resolveTakesRepoDir,
|
||||
TakesWriteError,
|
||||
} from '../takes-write.ts';
|
||||
|
||||
// --- v0.28: Takes ---
|
||||
|
||||
@@ -81,7 +94,7 @@ const takes_scorecard: Operation = {
|
||||
until: { type: 'string', description: 'Window end (YYYY-MM-DD)' },
|
||||
},
|
||||
handler: async (ctx, p) => {
|
||||
return ctx.engine.getScorecard(
|
||||
const card = await ctx.engine.getScorecard(
|
||||
{
|
||||
...sourceScopeOpts(ctx),
|
||||
holder: p.holder as string | undefined,
|
||||
@@ -91,6 +104,13 @@ const takes_scorecard: Operation = {
|
||||
},
|
||||
ctx.takesHoldersAllowList,
|
||||
);
|
||||
// [OV8/EV5] Resolver-provenance visibility: remote resolutions are
|
||||
// server-stamped resolved_by='mcp:<client>' (takes_resolve below), and
|
||||
// this coarse count keeps agent-resolved rows segregable from owner
|
||||
// ground truth. Op-layer SQL (plain executeRaw both engines share) —
|
||||
// no engine method added, parity untouched. Unfiltered by the window/
|
||||
// domain params (coarse by design; documented).
|
||||
return { ...card, mcp_resolved: await countMcpResolved(ctx) };
|
||||
},
|
||||
cliHints: { name: 'takes-scorecard' },
|
||||
};
|
||||
@@ -108,6 +128,9 @@ const takes_calibration: Operation = {
|
||||
bucket_size: { type: 'number', description: 'Bucket width in (0,1]; default 0.1' },
|
||||
},
|
||||
handler: async (ctx, p) => {
|
||||
// (The [OV8/EV5] mcp_resolved provenance count lives on takes_scorecard —
|
||||
// this op's wire shape is a bare bucket ARRAY, so decorating it would be
|
||||
// a breaking change; the scorecard is the segregation surface.)
|
||||
return ctx.engine.getCalibrationCurve(
|
||||
{
|
||||
...sourceScopeOpts(ctx),
|
||||
@@ -120,6 +143,39 @@ const takes_calibration: Operation = {
|
||||
cliHints: { name: 'takes-calibration' },
|
||||
};
|
||||
|
||||
/**
|
||||
* [OV8/EV5] Count of resolved take rows whose resolved_by carries the MCP
|
||||
* server-stamp prefix. Source-scoped + holder-allow-list-scoped like the
|
||||
* aggregates it decorates; unfiltered by window/domain (coarse segregation
|
||||
* signal, not a scorecard dimension).
|
||||
*/
|
||||
async function countMcpResolved(ctx: OperationContext): Promise<number> {
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
const where: string[] = [`t.resolved_at IS NOT NULL`, `t.resolved_by LIKE 'mcp:%'`];
|
||||
const params: unknown[] = [];
|
||||
if (scope.sourceIds && scope.sourceIds.length > 0) {
|
||||
params.push(scope.sourceIds);
|
||||
where.push(`p.source_id = ANY($${params.length}::text[])`);
|
||||
} else if (scope.sourceId) {
|
||||
params.push(scope.sourceId);
|
||||
where.push(`p.source_id = $${params.length}`);
|
||||
}
|
||||
if (ctx.takesHoldersAllowList) {
|
||||
params.push(ctx.takesHoldersAllowList);
|
||||
where.push(`t.holder = ANY($${params.length}::text[])`);
|
||||
}
|
||||
try {
|
||||
const rows = await ctx.engine.executeRaw<{ n: string }>(
|
||||
`SELECT count(*)::text AS n FROM takes t JOIN pages p ON p.id = t.page_id WHERE ${where.join(' AND ')}`,
|
||||
params,
|
||||
);
|
||||
return parseInt(rows[0]?.n ?? '0', 10);
|
||||
} catch {
|
||||
// Decoration only — never fail the aggregate over the provenance count.
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
const think: Operation = {
|
||||
name: 'think',
|
||||
description: 'Multi-hop synthesis across pages + takes + graph. Pulls relevant evidence and produces a cited answer with conflict + gap analysis.',
|
||||
@@ -187,13 +243,310 @@ const think: Operation = {
|
||||
remote_persisted_blocked: remote && (Boolean(p.save) || Boolean(p.take)),
|
||||
};
|
||||
},
|
||||
cliHints: { name: 'think', positional: ['question'] },
|
||||
// hidden: 'think' is in CLI_ONLY (src/cli.ts) — the richer runThinkCli
|
||||
// handler owns the CLI surface; a non-hidden hint here is dead (CLI_ONLY
|
||||
// wins at dispatch) and lies to the catalog.
|
||||
cliHints: { name: 'think', positional: ['question'], hidden: true },
|
||||
};
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI→MCP gap-closure wave — takes WRITE verbs. Before these, agents could
|
||||
// read the predictions ledger (takes_list/search/scorecard) but never record,
|
||||
// refine, resolve, or supersede a take. Backed by the same md-canonical
|
||||
// write-through core as the CLI (src/core/takes-write.ts): fence-derived row
|
||||
// numbers, markdown written first, the DB mirrored with the reconcile
|
||||
// primitive — and the markdown write is REQUIRED (no sync.repo_path on the
|
||||
// host → 'unavailable' with detail takes_mirror_unavailable), because a
|
||||
// DB-only row would be clobbered by the next md→DB reconcile.
|
||||
//
|
||||
// Trust model (ungated by design — the put_page precedent: writes are
|
||||
// consented via scope + the holder fence; publish gates cover owner-content
|
||||
// READ surfaces):
|
||||
// - Holder WRITE fence reuses the read allow-list, fail-closed: remote
|
||||
// effective list = ctx.takesHoldersAllowList ?? ['world'] (the stdio
|
||||
// default — stdio agents write world-held takes only; OAuth tokens can
|
||||
// grant more), [] = deny-all. add/supersede check the param holder;
|
||||
// update/resolve/supersede check the TARGET row's holder — and a fenced
|
||||
// row presents as not_found (same shape as a missing row), hiding content
|
||||
// and holder. Existence-by-count is accepted: row numbers are dense per
|
||||
// page, so takes_add's returned row_num reveals how many rows (including
|
||||
// private ones) exist — returning it is functionally required for later
|
||||
// update/resolve, and takes_list's visible-sequence gaps leak the same
|
||||
// count anyway [OV10/EV3].
|
||||
// - resolved_by is SERVER-STAMPED for remote callers (mcp:<clientId>) so
|
||||
// agent-resolved rows stay segregable from owner-resolved ground truth;
|
||||
// scorecard/calibration surface the mcp_resolved count [OV8/EV5].
|
||||
// - A future permissions.takes_write_holders key could split the read/write
|
||||
// axes if field use demands it.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TAKE_KINDS = ['fact', 'take', 'bet', 'hunch'] as const;
|
||||
/** Ops hold the MCP request at most this long waiting for the page lock. */
|
||||
const OP_LOCK_TIMEOUT_MS = 2000;
|
||||
|
||||
/** Remote callers get the read allow-list as the WRITE fence; local CLI is unfenced. */
|
||||
function takesWriteAllowList(ctx: OperationContext): readonly string[] | null {
|
||||
return ctx.remote !== false ? (ctx.takesHoldersAllowList ?? ['world']) : null;
|
||||
}
|
||||
|
||||
async function opBrainDir(ctx: OperationContext): Promise<string> {
|
||||
const dir = await resolveTakesRepoDir(ctx.engine);
|
||||
if (!dir) {
|
||||
const err = new OperationError(
|
||||
'unavailable',
|
||||
'Takes are markdown-canonical and this brain has no writable markdown repo configured.',
|
||||
'Configure sync.repo_path on the brain host, then retry.',
|
||||
);
|
||||
err.detail = 'takes_mirror_unavailable';
|
||||
throw err;
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function mapTakesWriteError(err: unknown): never {
|
||||
if (err instanceof TakesWriteError) {
|
||||
switch (err.code) {
|
||||
case 'page_not_found':
|
||||
throw new OperationError('page_not_found', err.message, 'Sync the brain first, or check the slug/source.');
|
||||
case 'row_not_found':
|
||||
// Fenced rows deliberately share this shape (no-existence-leak of
|
||||
// content/holder — see the trust-model comment above).
|
||||
throw new OperationError('not_found', err.message);
|
||||
case 'holder_denied': {
|
||||
const e = new OperationError('permission_denied', err.message,
|
||||
"Ask the brain owner to widen this caller's takes-holder allow-list.");
|
||||
e.detail = 'holder_not_in_allowlist';
|
||||
throw e;
|
||||
}
|
||||
case 'mirror_unavailable': {
|
||||
const e = new OperationError('unavailable', err.message,
|
||||
'Configure sync.repo_path on the brain host, then retry.');
|
||||
e.detail = 'takes_mirror_unavailable';
|
||||
throw e;
|
||||
}
|
||||
case 'page_locked': {
|
||||
const e = new OperationError('unavailable', err.message, 'Retry shortly.');
|
||||
e.detail = 'retryable';
|
||||
throw e;
|
||||
}
|
||||
case 'already_resolved':
|
||||
throw new OperationError('invalid_params', err.message, err.hint ?? 'Resolved takes are immutable; supersede instead.');
|
||||
case 'fence_unparsed':
|
||||
case 'row_inactive':
|
||||
case 'no_fields':
|
||||
case 'invalid_input':
|
||||
throw new OperationError('invalid_params', err.message, err.hint);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
/**
|
||||
* P1-4/F4: the markdown write already succeeded; a non-empty `mirror_warning`
|
||||
* means only the DB mirror deferred to the next reconcile. Surface it so the
|
||||
* agent knows the durable row is on disk and MUST NOT retry.
|
||||
*/
|
||||
function mirrorWarnFields(mirror: { mirror_warning?: string }): Record<string, string> {
|
||||
return mirror.mirror_warning
|
||||
? { mirror_warning: `row written to markdown; DB mirror deferred to reconcile: ${mirror.mirror_warning}` }
|
||||
: {};
|
||||
}
|
||||
|
||||
const takes_add: Operation = {
|
||||
name: 'takes_add',
|
||||
description:
|
||||
'Record a take (typed claim) on a page: fact / take / bet / hunch, with a holder (who ' +
|
||||
'holds the belief: world, people/<slug>, companies/<slug>, or brain), weight 0..1, and ' +
|
||||
'optional source/since date. Writes the markdown takes fence first (markdown is ' +
|
||||
'canonical) and mirrors to the DB. Remote callers can only write holders in their ' +
|
||||
'allow-list (stdio default: world).',
|
||||
params: {
|
||||
slug: { type: 'string', required: true, description: 'Page slug to attach the take to (page must exist).' },
|
||||
claim: { type: 'string', required: true, description: 'The claim text (one line).' },
|
||||
kind: { type: 'string', required: true, enum: [...TAKE_KINDS], description: 'Claim type. Base kinds only; pack-extended kinds are a filed follow-up.' },
|
||||
holder: { type: 'string', required: true, description: "Who HOLDS this belief (said/clearly implied it): world | people/<slug> | companies/<slug> | brain. Remote callers: must be in the caller's takes-holder allow-list." },
|
||||
weight: { type: 'number', required: false, description: 'Confidence 0..1 (default 0.5; clamped server-side).' },
|
||||
source: { type: 'string', required: false, description: 'Where the claim came from (free text).' },
|
||||
since: { type: 'string', required: false, description: "When the belief started ('YYYY-MM' or 'YYYY-MM-DD')." },
|
||||
},
|
||||
scope: 'write',
|
||||
mutating: true,
|
||||
area: 'takes',
|
||||
handler: async (ctx, p) => {
|
||||
const slug = p.slug as string;
|
||||
enforceClientSlugFence(ctx, slug, 'takes_add');
|
||||
validatePageSlug(slug); // defense-in-depth, matching put_page
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'takes_add', slug };
|
||||
const brainDir = await opBrainDir(ctx);
|
||||
try {
|
||||
const { rowNum, mirror } = await addTakeToPage(
|
||||
{ engine: ctx.engine, slug, brainDir, sourceId: ctx.sourceId, allowList: takesWriteAllowList(ctx), lockTimeoutMs: OP_LOCK_TIMEOUT_MS },
|
||||
{
|
||||
claim: p.claim as string,
|
||||
kind: p.kind as string,
|
||||
holder: p.holder as string,
|
||||
weight: p.weight as number | undefined,
|
||||
source: p.source as string | undefined,
|
||||
sinceDate: p.since as string | undefined,
|
||||
},
|
||||
);
|
||||
return { slug, row_num: rowNum, holder: p.holder, mirror_written: true, ...mirrorWarnFields(mirror) };
|
||||
} catch (err) {
|
||||
mapTakesWriteError(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const takes_update: Operation = {
|
||||
name: 'takes_update',
|
||||
description:
|
||||
'Update a take\'s mutable fields (weight, source, since date). Claim/kind/holder are ' +
|
||||
'immutable — supersede instead. Markdown-canonical: the target row must exist in the ' +
|
||||
'page\'s takes fence. Remote callers can only touch rows whose holder is in their ' +
|
||||
'allow-list; other rows present as not_found.',
|
||||
params: {
|
||||
slug: { type: 'string', required: true, description: 'Page slug.' },
|
||||
row_num: { type: 'number', required: true, description: 'Take row number on the page (from takes_list).' },
|
||||
weight: { type: 'number', required: false, description: 'New confidence 0..1.' },
|
||||
source: { type: 'string', required: false, description: 'New source text.' },
|
||||
since: { type: 'string', required: false, description: "New since date ('YYYY-MM' or 'YYYY-MM-DD')." },
|
||||
},
|
||||
scope: 'write',
|
||||
mutating: true,
|
||||
area: 'takes',
|
||||
handler: async (ctx, p) => {
|
||||
const slug = p.slug as string;
|
||||
enforceClientSlugFence(ctx, slug, 'takes_update');
|
||||
validatePageSlug(slug); // defense-in-depth, matching put_page
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'takes_update', slug, row_num: p.row_num };
|
||||
const brainDir = await opBrainDir(ctx);
|
||||
try {
|
||||
const { rowNum, mirror } = await updateTakeOnPage(
|
||||
{ engine: ctx.engine, slug, brainDir, sourceId: ctx.sourceId, allowList: takesWriteAllowList(ctx), lockTimeoutMs: OP_LOCK_TIMEOUT_MS },
|
||||
p.row_num as number,
|
||||
{
|
||||
weight: p.weight as number | undefined,
|
||||
source: p.source as string | undefined,
|
||||
sinceDate: p.since as string | undefined,
|
||||
},
|
||||
);
|
||||
return { slug, row_num: rowNum, updated: true, ...mirrorWarnFields(mirror) };
|
||||
} catch (err) {
|
||||
mapTakesWriteError(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const takes_supersede: Operation = {
|
||||
name: 'takes_supersede',
|
||||
description:
|
||||
'Supersede a take with a replacement claim: the old row is struck through (kept for ' +
|
||||
'archaeology), the replacement appends at the next fence row number. Kind/holder inherit ' +
|
||||
'from the target row unless overridden; unset weight decays the target\'s by 0.1. ' +
|
||||
'Markdown-canonical; remote holder fencing as in takes_update.',
|
||||
params: {
|
||||
slug: { type: 'string', required: true, description: 'Page slug.' },
|
||||
row_num: { type: 'number', required: true, description: 'Row number of the take being superseded.' },
|
||||
claim: { type: 'string', required: true, description: 'The replacement claim text.' },
|
||||
kind: { type: 'string', required: false, enum: [...TAKE_KINDS], description: 'Override kind (default: inherit from the target row).' },
|
||||
holder: { type: 'string', required: false, description: 'Override holder (default: inherit). Remote callers: an override must be in the allow-list.' },
|
||||
weight: { type: 'number', required: false, description: "Replacement confidence 0..1 (default: target's weight - 0.1)." },
|
||||
source: { type: 'string', required: false, description: 'Source for the replacement.' },
|
||||
since: { type: 'string', required: false, description: 'Since date for the replacement.' },
|
||||
},
|
||||
scope: 'write',
|
||||
mutating: true,
|
||||
area: 'takes',
|
||||
handler: async (ctx, p) => {
|
||||
const slug = p.slug as string;
|
||||
enforceClientSlugFence(ctx, slug, 'takes_supersede');
|
||||
validatePageSlug(slug); // defense-in-depth, matching put_page
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'takes_supersede', slug, row_num: p.row_num };
|
||||
const brainDir = await opBrainDir(ctx);
|
||||
try {
|
||||
const { oldRow, newRow, mirror } = await supersedeTakeOnPage(
|
||||
{ engine: ctx.engine, slug, brainDir, sourceId: ctx.sourceId, allowList: takesWriteAllowList(ctx), lockTimeoutMs: OP_LOCK_TIMEOUT_MS },
|
||||
p.row_num as number,
|
||||
{
|
||||
claim: p.claim as string,
|
||||
kind: p.kind as string | undefined,
|
||||
holder: p.holder as string | undefined,
|
||||
weight: p.weight as number | undefined,
|
||||
source: p.source as string | undefined,
|
||||
sinceDate: p.since as string | undefined,
|
||||
},
|
||||
);
|
||||
return { slug, old_row: oldRow, new_row: newRow, ...mirrorWarnFields(mirror) };
|
||||
} catch (err) {
|
||||
mapTakesWriteError(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const takes_resolve: Operation = {
|
||||
name: 'takes_resolve',
|
||||
description:
|
||||
'Resolve a take: quality correct / incorrect / partial / unresolvable, with optional ' +
|
||||
'evidence text and measured value/unit. Resolutions feed the calibration scorecard. ' +
|
||||
'Remote callers: resolved_by is SERVER-STAMPED as mcp:<client> (any passed value is ' +
|
||||
'ignored) so agent resolutions stay segregable from owner ground truth; the target row ' +
|
||||
'must be in the caller\'s holder allow-list. Markdown-canonical.',
|
||||
params: {
|
||||
slug: { type: 'string', required: true, description: 'Page slug.' },
|
||||
row_num: { type: 'number', required: true, description: 'Take row number to resolve.' },
|
||||
quality: { type: 'string', required: true, enum: ['correct', 'incorrect', 'partial', 'unresolvable'], description: 'Resolution verdict.' },
|
||||
evidence: { type: 'string', required: false, description: 'What evidence resolved this (free text).' },
|
||||
value: { type: 'number', required: false, description: 'Measured value, when the claim was quantitative.' },
|
||||
unit: { type: 'string', required: false, description: 'Unit for value (usd | pct | count | ...).' },
|
||||
resolved_by: { type: 'string', required: false, description: 'Resolver identity. Remote callers: SERVER-STAMPED mcp:<client>, client value ignored. Local default: the configured owner holder.' },
|
||||
},
|
||||
scope: 'write',
|
||||
mutating: true,
|
||||
area: 'takes',
|
||||
handler: async (ctx, p) => {
|
||||
const slug = p.slug as string;
|
||||
enforceClientSlugFence(ctx, slug, 'takes_resolve');
|
||||
validatePageSlug(slug); // defense-in-depth, matching put_page
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'takes_resolve', slug, row_num: p.row_num };
|
||||
const brainDir = await opBrainDir(ctx);
|
||||
// CV6 posture: remote resolutions are provenance-stamped server-side —
|
||||
// clamp + sanitize the client id: hostile DCR client names must not carry
|
||||
// newlines/pipes into the markdown fence, and must not bloat the column.
|
||||
let resolvedBy: string;
|
||||
if (ctx.remote !== false) {
|
||||
const id = (ctx.auth?.clientId ?? ctx.transport ?? 'remote').replace(/[^\w.:-]/g, '_').slice(0, 64);
|
||||
resolvedBy = `mcp:${id}`;
|
||||
} else if (typeof p.resolved_by === 'string' && p.resolved_by.length > 0) {
|
||||
resolvedBy = p.resolved_by;
|
||||
} else {
|
||||
const { resolveOwnerHolder } = await import('../owner-holder.ts');
|
||||
resolvedBy = resolveOwnerHolder({ configValue: await ctx.engine.getConfig('emotional_weight.user_holder') });
|
||||
}
|
||||
try {
|
||||
const { rowNum, quality, mirror } = await resolveTakeOnPage(
|
||||
{ engine: ctx.engine, slug, brainDir, sourceId: ctx.sourceId, allowList: takesWriteAllowList(ctx), lockTimeoutMs: OP_LOCK_TIMEOUT_MS },
|
||||
p.row_num as number,
|
||||
{
|
||||
quality: p.quality as 'correct' | 'incorrect' | 'partial' | 'unresolvable',
|
||||
evidence: p.evidence as string | undefined,
|
||||
value: p.value as number | undefined,
|
||||
unit: p.unit as string | undefined,
|
||||
resolvedBy,
|
||||
},
|
||||
);
|
||||
return { slug, row_num: rowNum, quality, resolved_by: resolvedBy, ...mirrorWarnFields(mirror) };
|
||||
} catch (err) {
|
||||
mapTakesWriteError(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Ops in EXACTLY the canonical `operations` array order: the v0.28 trio
|
||||
// (takes_list, takes_search, think), then the v0.30 calibration aggregates.
|
||||
// (takes_list, takes_search, think), the v0.30 calibration aggregates, then
|
||||
// the gap-closure write verbs.
|
||||
export const takesOperations: Operation[] = [
|
||||
takes_list, takes_search, think,
|
||||
takes_scorecard, takes_calibration,
|
||||
takes_add, takes_update, takes_resolve, takes_supersede,
|
||||
];
|
||||
|
||||
@@ -78,7 +78,18 @@ export class SlugRegistryError extends Error {
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u');
|
||||
|
||||
export class SlugRegistry {
|
||||
constructor(private engine: BrainEngine) {}
|
||||
/**
|
||||
* `sourceId` scopes every existence probe to the SAME source the paired
|
||||
* putPage will write to (engine.putPage defaults to 'default' when unset).
|
||||
* Pre-fix the probes were UNSCOPED — getPage matched a slug in ANY source,
|
||||
* so a slug taken only in source B forced a spurious disambiguation (or a
|
||||
* false isFree=false) for a write that was going to land in source A.
|
||||
*/
|
||||
constructor(private engine: BrainEngine, private sourceId?: string) {}
|
||||
|
||||
private scope(): { sourceId: string } {
|
||||
return { sourceId: this.sourceId ?? 'default' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new slug, or disambiguate if taken. Checks engine.getPage(slug)
|
||||
@@ -93,7 +104,7 @@ export class SlugRegistry {
|
||||
}
|
||||
|
||||
// Fast path: desired is free
|
||||
const existing = await this.engine.getPage(desiredSlug);
|
||||
const existing = await this.engine.getPage(desiredSlug, this.scope());
|
||||
if (!existing) {
|
||||
return { slug: desiredSlug, exact: true };
|
||||
}
|
||||
@@ -110,7 +121,7 @@ export class SlugRegistry {
|
||||
// append-numeric disambiguation: start at 2 (matches "alice-smith" → "alice-smith-2")
|
||||
for (let n = 2; n <= maxDisambiguator; n++) {
|
||||
const candidate = `${desiredSlug}-${n}`;
|
||||
const conflict = await this.engine.getPage(candidate);
|
||||
const conflict = await this.engine.getPage(candidate, this.scope());
|
||||
if (!conflict) {
|
||||
return { slug: candidate, exact: false, disambiguator: n };
|
||||
}
|
||||
@@ -129,7 +140,7 @@ export class SlugRegistry {
|
||||
*/
|
||||
async isFree(slug: string): Promise<boolean> {
|
||||
if (!SLUG_RE.test(slug)) return false;
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
return !existing;
|
||||
}
|
||||
|
||||
|
||||
+35
-12
@@ -37,6 +37,15 @@ export interface BrainWriterOptions {
|
||||
* follow-on release after soak).
|
||||
*/
|
||||
strictMode?: StrictMode;
|
||||
/**
|
||||
* Source every read AND write in this writer targets. Default 'default'
|
||||
* (matches engine.putPage's schema default). Pre-fix the writer's reads
|
||||
* were UNSCOPED (first slug match across ANY source) while its writes
|
||||
* landed in 'default' — the exact unscoped-check/scoped-write bug class:
|
||||
* setCompiledTruth could read source B's page and clobber the default
|
||||
* source's row with it.
|
||||
*/
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface EntityInput {
|
||||
@@ -128,8 +137,14 @@ class WriteTxImpl implements WriteTx {
|
||||
constructor(
|
||||
private engine: BrainEngine,
|
||||
public readonly context: ResolverContext,
|
||||
private sourceId?: string,
|
||||
) {
|
||||
this.slugRegistry = new SlugRegistry(engine);
|
||||
this.slugRegistry = new SlugRegistry(engine, sourceId);
|
||||
}
|
||||
|
||||
/** Read+write scope: mirrors engine.putPage's implicit 'default'. */
|
||||
private scope(): { sourceId: string } {
|
||||
return { sourceId: this.sourceId ?? 'default' };
|
||||
}
|
||||
|
||||
async createEntity(input: EntityInput): Promise<string> {
|
||||
@@ -163,18 +178,18 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: input.compiledTruth,
|
||||
timeline: input.timeline ?? '',
|
||||
frontmatter: input.frontmatter ?? {},
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
return slug;
|
||||
}
|
||||
|
||||
async appendTimeline(slug: string, entry: TimelineInput): Promise<void> {
|
||||
await this.engine.addTimelineEntry(slug, entry); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface — output gets fenced into pages via putPage in the same transaction
|
||||
await this.engine.addTimelineEntry(slug, entry, this.scope()); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface — output gets fenced into pages via putPage in the same transaction
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async setCompiledTruth(slug: string, body: string): Promise<void> {
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
if (!existing) throw new WriteError('invalid_input', `setCompiledTruth: page not found: ${slug}`);
|
||||
await this.engine.putPage(slug, {
|
||||
type: existing.type,
|
||||
@@ -182,12 +197,12 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: body,
|
||||
timeline: existing.timeline,
|
||||
frontmatter: existing.frontmatter,
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async setFrontmatterField(slug: string, key: string, value: unknown): Promise<void> {
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
if (!existing) throw new WriteError('invalid_input', `setFrontmatterField: page not found: ${slug}`);
|
||||
const nextFm = { ...existing.frontmatter, [key]: value };
|
||||
await this.engine.putPage(slug, {
|
||||
@@ -196,21 +211,26 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: existing.compiled_truth,
|
||||
timeline: existing.timeline,
|
||||
frontmatter: nextFm,
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async putRawData(slug: string, source: string, data: object): Promise<void> {
|
||||
await this.engine.putRawData(slug, source, data);
|
||||
await this.engine.putRawData(slug, source, data, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async addLink(from: string, to: string, context?: string, linkType?: string): Promise<void> {
|
||||
await this.engine.addLink(from, to, context, linkType); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface
|
||||
// Both endpoints scoped to this writer's source — synthesize-phase links
|
||||
// are within-source by definition, and unscoped endpoints resolve against
|
||||
// 'default'-source rows (wrong page or missing) in a scoped writer.
|
||||
const sid = this.scope().sourceId;
|
||||
const linkScope = { fromSourceId: sid, toSourceId: sid };
|
||||
await this.engine.addLink(from, to, context, linkType, undefined, undefined, undefined, linkScope); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface
|
||||
// Reverse back-link — both directions inside the same outer transaction.
|
||||
// Uses 'backlink' label on the reverse if no linkType was specified so
|
||||
// the reverse is distinguishable from the forward semantic type.
|
||||
await this.engine.addLink(to, from, context, linkType ? `${linkType}_back` : 'backlink'); // gbrain-allow-direct-insert: BrainWriter synthesize-phase reverse back-link in the same transaction as the forward addLink above
|
||||
await this.engine.addLink(to, from, context, linkType ? `${linkType}_back` : 'backlink', undefined, undefined, undefined, linkScope); // gbrain-allow-direct-insert: BrainWriter synthesize-phase reverse back-link in the same transaction as the forward addLink above
|
||||
this.touchedSlugs.add(from);
|
||||
this.touchedSlugs.add(to);
|
||||
}
|
||||
@@ -223,12 +243,14 @@ class WriteTxImpl implements WriteTx {
|
||||
export class BrainWriter {
|
||||
private validators: PageValidator[] = [];
|
||||
private strictMode: StrictMode;
|
||||
private sourceId?: string;
|
||||
|
||||
constructor(
|
||||
private engine: BrainEngine,
|
||||
opts: BrainWriterOptions = {},
|
||||
) {
|
||||
this.strictMode = opts.strictMode ?? 'lint';
|
||||
this.sourceId = opts.sourceId;
|
||||
}
|
||||
|
||||
register(validator: PageValidator): void {
|
||||
@@ -247,14 +269,15 @@ export class BrainWriter {
|
||||
|
||||
let report: ValidationReport | null = null;
|
||||
|
||||
const strictSourceId = this.sourceId;
|
||||
const txResult = await this.engine.transaction(async (txEngine) => {
|
||||
const tx = new WriteTxImpl(txEngine, ctx);
|
||||
const tx = new WriteTxImpl(txEngine, ctx, strictSourceId);
|
||||
const result = await fn(tx);
|
||||
|
||||
// Validators run before the outer transaction commits.
|
||||
if (strict !== 'off') {
|
||||
report = await runValidators(txEngine, validators, tx.touchedSlugs, {
|
||||
sourceId: 'default',
|
||||
sourceId: strictSourceId ?? 'default',
|
||||
});
|
||||
// `ctx.logger.info` would be nice but keep validator behavior uniform
|
||||
// regardless of strict/lint mode. Caller inspects the report.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user