mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 18:02:30 +00:00
Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fbc73a23f | ||
|
|
877d1ca479 | ||
|
|
0b3bafeb06 | ||
|
|
506cfbf787 | ||
|
|
a944a7a6d2 | ||
|
|
70202a24f9 | ||
|
|
bf51bf500f | ||
|
|
6624f1a53b | ||
|
|
e3fd862caf | ||
|
|
70ed8060da | ||
|
|
3d203b7258 | ||
|
|
ee8d48e1b3 | ||
|
|
bf0b28cd64 | ||
|
|
86f0da5eb9 | ||
|
|
b8d07a0850 | ||
|
|
f3931df6a4 |
@@ -1,17 +1,33 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.11.0",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": { "name": "Garry Tan", "url": "https://github.com/garrytan" },
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": ["memory", "knowledge-base", "mcp", "search", "agent", "brain", "pgvector"],
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/.agents/gbrain-launcher",
|
||||
"args": ["serve", "--surface", "starter", "--source-guard"],
|
||||
"args": [
|
||||
"serve",
|
||||
"--surface",
|
||||
"starter",
|
||||
"--source-guard"
|
||||
],
|
||||
"cwd": "${CLAUDE_PLUGIN_ROOT}"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.11.0",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": { "name": "Garry Tan", "url": "https://github.com/garrytan" },
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": ["memory", "knowledge-base", "mcp", "search", "agent", "brain", "pgvector"],
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": "./.codex-plugin/mcp.json",
|
||||
"interface": {
|
||||
@@ -15,7 +26,10 @@
|
||||
"longDescription": "GBrain wires a personal knowledge brain into every session: hybrid keyword+vector search, entity graph traversal, synthesis, and memory your agent writes itself — served on the starter MCP surface (the seven memory verbs plus the daily-driver brain ops). Bundles the curated brain-first skill set: setup (walks install + gbrain init), cold-start day-one brain filling, ingest, query, briefing, upgrade, and more. Requires the gbrain CLI (bun install -g github:garrytan/gbrain#latest-stable) and a brain (gbrain init); the bundled setup skill walks the rest. Unix (macOS/Linux) only.",
|
||||
"developerName": "Garry Tan",
|
||||
"category": "Productivity",
|
||||
"capabilities": ["Interactive", "Write"],
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/garrytan/gbrain",
|
||||
"defaultPrompt": [
|
||||
"Search my brain, recall context across sessions, and write new memory as we work"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-runbook-stamp: 0.46.10.0 -->
|
||||
<!-- gbrain-runbook-stamp: 0.46.11.0 -->
|
||||
<!-- This stamp must equal the VERSION file at every release; CI enforces it
|
||||
(scripts/check-bootstrap-tag.sh). `gbrain bootstrap status` compares it to
|
||||
the installed binary and warns on skew. -->
|
||||
|
||||
@@ -2,6 +2,102 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.46.11.0] - 2026-08-16
|
||||
|
||||
**Five operational failures from live production brains, fixed at the root.**
|
||||
A backlink auto-fix that could corrupt a page's frontmatter, a job queue that
|
||||
grew a multi-thousand-job backlog with no admission control and no alarm,
|
||||
junk filenames that imported as plausible-looking pages and polluted search,
|
||||
a read/write source-scoping asymmetry that misrouted pages in multi-source
|
||||
brains, and frontmatter types that silently filed into unexpected
|
||||
directories. Each fix ships with its regression pinned and a discovery
|
||||
surface so the same failure can't build up silently again.
|
||||
|
||||
### Added
|
||||
- **Queue admission control for background agents.** Identical parentless
|
||||
`subagent` submits now coalesce onto the existing waiting job (same owner
|
||||
lane, payload, and execution options — the response carries `coalesced:
|
||||
true` so callers can tell); jobs still waiting after 48 hours are cancelled
|
||||
with an auditable reason instead of queueing forever (`gbrain config set
|
||||
minions.ttl_waiting_hours.<name> <hours|0>` to tune or disable); and an
|
||||
optional per-type waiting quota (`minions.quota_max_waiting.<name>`,
|
||||
off by default) rejects new submits with a structured, retryable error once
|
||||
a backlog cap is hit — exact even under concurrent submitters. Everything
|
||||
disables at once with `GBRAIN_MINIONS_ADMISSION=0`.
|
||||
- **Warn-before-act for the new waiting-TTL.** The first sweep never fires
|
||||
cold: the worker (and `gbrain upgrade`) print a one-time notice with the
|
||||
affected-job count, then hold a one-hour grace window before the first
|
||||
cancellation so there's real time to tune or opt out.
|
||||
- **Divergent-queue alarms.** `gbrain jobs stats` gains Drained/Waiting
|
||||
columns, a per-type `DIVERGENT QUEUE` scream when intake structurally
|
||||
exceeds completions (with the exact config command to cap it), a
|
||||
waiting-TTL 24h cancellation line, and a `--json` document; `gbrain
|
||||
doctor`'s queue health check surfaces the same findings for cron
|
||||
topologies. TTL cancellations are never counted as useful drain.
|
||||
- **Stored-type visibility.** Sync and import now warn once per run when
|
||||
explicit frontmatter types are aliases or undeclared in the active schema
|
||||
pack (aggregated counts ride the sync result and the `--json` envelope for
|
||||
worker topologies; silence with `schema.type_warnings false`), and `gbrain
|
||||
schema lint` gains two data-plane rules that catch the existing corpus,
|
||||
scoped per source.
|
||||
- **`gbrain quarantine clear --source-id`** — clearing a slug that exists in
|
||||
multiple sources now errors with the source list instead of picking one
|
||||
arbitrarily.
|
||||
- **`malformed_path_pages` doctor check** — finds previously ingested pages
|
||||
backed by junk filenames and says exactly which are sweepable versus which
|
||||
need a rename.
|
||||
- A shared atomic file writer (`src/core/atomic-write.ts`): unique temp
|
||||
sibling, full-write loop, fsync, on-disk verification callback, mode
|
||||
preservation past the umask, and parent-directory fsync after the rename.
|
||||
|
||||
### Fixed
|
||||
- **`check-backlinks fix` can no longer corrupt frontmatter.** The timeline
|
||||
inserter now computes the body offset from the canonical frontmatter
|
||||
parser (never matching headings inside YAML), validates the page before
|
||||
and after the edit, writes atomically with an on-disk verify, takes the
|
||||
per-page lock, and isolates per-file errors so one bad page can't poison a
|
||||
batch. Pages with pre-existing broken frontmatter are skipped and reported
|
||||
instead of made worse.
|
||||
- **Junk filenames no longer import.** Markdown paths containing brackets or
|
||||
any path containing control characters are rejected at sync, import, and
|
||||
the direct file-import defense (before any slug is minted), with the skip
|
||||
visibly reported on every route — including dry runs, directory imports,
|
||||
and syncs whose only changes were malformed files. Previously ingested
|
||||
junk rows are swept by the next full sync; legitimately bracket-named
|
||||
markdown from older releases is preserved (rename to re-import), and
|
||||
code-strategy sources keep indexing framework layouts like `app/[id]/`.
|
||||
- **Source-scoped reads now mirror their writes.** The existence-check/write
|
||||
asymmetry that misrouted pages in multi-source brains is closed across the
|
||||
writer transaction (pages, links, raw data, validators, slug registry),
|
||||
file import, image import, code reindex, and integrity repair — enforced
|
||||
going forward by a CI guard, with unscoped reads made deterministic
|
||||
(default source first) in both engines.
|
||||
- Waiting-TTL cancellations flow through the canonical cancel path so
|
||||
aggregator parents always resolve, reasons stamp only the jobs that
|
||||
actually expired, and a cancelled child frees its idempotency slot.
|
||||
- Interactive `gbrain agent run` prints `coalesced` (with the matched job id)
|
||||
instead of a false `submitted` when admission coalescing matched an
|
||||
existing waiting job; the remote submit surface returns the same signal and
|
||||
maps quota rejections to a structured `rate_limited` error.
|
||||
- Job names and frontmatter-derived type strings are sanitized before
|
||||
terminal output, and copy-pasteable remediation hints only embed values
|
||||
that are shell-safe tokens.
|
||||
- Page-lock acquisition is now exclusive-create, so two processes reclaiming
|
||||
a stale lock can no longer both proceed and lose one side's writes.
|
||||
- The advisor's stalled-jobs recommendation and the schema-lint retype hint
|
||||
now point at commands that exist.
|
||||
|
||||
### To take advantage of v0.46.11.0
|
||||
Upgrade and restart the worker (`bun install -g github:garrytan/gbrain#latest-stable
|
||||
&& gbrain upgrade`). The one-time waiting-TTL notice will print with your
|
||||
affected-job count and hold a one-hour grace window — tune with `gbrain
|
||||
config set minions.ttl_waiting_hours.subagent <hours|0>` before the first
|
||||
sweep if 48h isn't right for you. Then check `gbrain jobs stats`: if you see
|
||||
a `DIVERGENT QUEUE` scream, the printed `minions.quota_max_waiting.<name>`
|
||||
command is the opt-in cap. Existing junk-filename pages are removed by your
|
||||
next full `gbrain sync` (files stay on disk; rename a file to re-import its
|
||||
content), and `gbrain doctor` will name anything that needs a manual rename.
|
||||
|
||||
## [0.46.10.0] - 2026-08-16
|
||||
|
||||
**Switching embedding and reranking providers is now one guess-free
|
||||
|
||||
@@ -58,7 +58,12 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
sites; `ctx.remote !== false` for untrust-unless-explicit-false). Don't default it falsy.
|
||||
- **Source isolation.** Every read-side op routes through `sourceScopeOpts(ctx)`; precedence
|
||||
is federated array (`ctx.auth.allowedSources`) > scalar (`ctx.sourceId`) > nothing. Don't
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak.
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak. Corollary
|
||||
(unscoped-check/scoped-write): `engine.getPage` with no opts matches ANY source while
|
||||
`putPage` defaults to `'default'` — an existence check + write pair must scope the read
|
||||
to the write's source (`getPage(slug, { sourceId: x ?? 'default' })`). Guarded by
|
||||
`scripts/check-getpage-scoped-write.mjs` (opt-out marker
|
||||
`gbrain-allow-unscoped-getpage` for read-only first-match sites).
|
||||
- **JSONB: never `JSON.stringify` into a `::jsonb` cast.** postgres.js double-encodes it (a jsonb
|
||||
string scalar); PGLite hides the bug. This bites BOTH spellings — the template form
|
||||
(`${JSON.stringify(x)}::jsonb`) AND the positional form (`executeRaw(\`…$N::jsonb\`, [JSON.stringify(x)])`,
|
||||
|
||||
@@ -1,5 +1,98 @@
|
||||
# TODOS
|
||||
|
||||
## Five-issue fix wave follow-ups (backlinks corruption / malformed paths / type warnings / getPage scoping / queue admission)
|
||||
|
||||
- [ ] **P2 — migrate the remaining fs writers to core/atomic-write.** **What:**
|
||||
`src/core/skillopt/apply-edits.ts` (atomicWrite, leaks tmp on write error),
|
||||
`src/core/write-through.ts` (own tmp+rename), `src/commands/lint.ts:~526`
|
||||
(bare writeFileSync in runLintCore) move onto `src/core/atomic-write.ts`
|
||||
(unique tmp + fsync + mode preservation + optional on-disk verify). Include
|
||||
page-lock unification: write-through's render does NOT take withPageLock, so
|
||||
the backlinks-vs-render lost-update race is only half-closed (backlinks
|
||||
locks; render doesn't). **Why:** four hand-rolled copies drift; the shared
|
||||
helper is strictly stronger. **Effort:** M. **Priority:** P2.
|
||||
- [ ] **P3 — relocate/retire skillopt's splitFrontmatter.** **What:** either
|
||||
move it to core/markdown.ts next to frontmatterBodyOffset or port its one
|
||||
SKILL.md caller onto the canonical helper (skillopt's regex is LF-at-byte-0
|
||||
only; the canonical one handles leading blanks + CRLF). **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — admission/stats indexes if hot.** **What:** expression index on
|
||||
`(name, (data->>'__param_hash')) WHERE status='waiting'` for the coalesce
|
||||
probe + `(name, created_at)` for the per-type stats aggregates, when
|
||||
minion_jobs exceeds ~100k rows. Same family as the buildQueueDepths perf
|
||||
note (status.ts) and the completed-recency probe TODO below. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P2 — getPage type-boundary redesign (the durable fix behind the
|
||||
guard).** **What:** make source scope explicit at the TYPE level — required
|
||||
scope param or an explicit ALL_SOURCES sentinel on `engine.getPage`, so an
|
||||
unscoped read is unrepresentable instead of merely linted
|
||||
(check-getpage-scoped-write.mjs is the interim guard; the default-first
|
||||
ORDER BY makes today's unscoped reads deterministic). ~78 call sites.
|
||||
**Effort:** L. **Priority:** P2.
|
||||
- [ ] **P2 — per-name claim fairness / lane isolation.** **What:** the
|
||||
admission wave (coalescing/TTL/quota) is deliberately submit-side only;
|
||||
claim order remains global FIFO per queue (`queue.ts` claim ORDER BY), so
|
||||
one divergent type still starves same-queue siblings until TTL/quota bites.
|
||||
A per-name claim budget or weighted claim is the drain-side primitive.
|
||||
**Effort:** L. **Priority:** P2.
|
||||
- [ ] **P3 — jobs stats divergence: per-queue scoping option.** **What:**
|
||||
the DIVERGENT scream computes name-global (matches quota semantics); a
|
||||
`--queue`-scoped variant would help multi-queue operators localize the
|
||||
producer. **Effort:** S. **Priority:** P3.
|
||||
- [ ] **P2 — requeue surface for waiting-TTL-cancelled jobs.** **What:**
|
||||
`jobs retry` targets failed/dead only; a TTL-cancelled row (error_text
|
||||
prefix `waiting_ttl_expired`) that turns out to have been wanted needs a
|
||||
`jobs requeue` (or a retry carve-out gated on that prefix) instead of
|
||||
hand-resubmitting. The data survives (cancelled rows keep payloads +
|
||||
free their idempotency keys), so this is purely a CLI surface. **Effort:**
|
||||
S. **Priority:** P2. (Pre-landing data-migration review, five-issue wave.)
|
||||
- [ ] **P2 — dream-path quota-degradation integration tests.** **What:**
|
||||
live-queue integration tests for the QueueQuotaExceededError consumers:
|
||||
cycle patterns → `skipped('admission_quota')`, synthesize → quota latch
|
||||
(one skip per remaining transcript, stop submitting), agent fanout →
|
||||
whole-tree cancel + exit 1. Unit seams exist (isQueueQuotaExceededError
|
||||
is pinned); what's missing is the end-to-end phase behavior under a
|
||||
1-quota config. **Effort:** M. **Priority:** P2.
|
||||
- [ ] **P3 — coalesce advisory-lock concurrency e2e.** **What:** real-PG
|
||||
e2e slamming N concurrent identical parentless submits → exactly one row
|
||||
(the advisory lock serializes (name, queue, hash)); PGLite can't prove
|
||||
this (single connection). Home: the DATABASE_URL-gated e2e lane.
|
||||
**Effort:** S. **Priority:** P3.
|
||||
- [ ] **P3 — consolidate the stable-stringify triplets.** **What:**
|
||||
`admission.ts` (param hash), plus the two earlier canonical-JSON copies
|
||||
(op-checkpoint hashing, cli-options) each roll their own sorted-key
|
||||
stringify; one `core/canonical-json.ts` would do. Hash-compat note: the
|
||||
admission copy feeds persisted `__param_hash` values — a behavior-change
|
||||
regression there just disables old-row coalescing (forward-safe), but
|
||||
keep the sorted-key semantics bit-identical anyway. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — reconcile lane: quarantine-not-delete option for malformed-path
|
||||
rows + doctor hint nuance.** **What:** full-sync reconcile hard-deletes
|
||||
poisoned rows (consistent with 'strategy' semantics); a
|
||||
`--quarantine-malformed` alternative would preserve rows for triage. Also
|
||||
the malformed_path_pages doctor hint could distinguish rows whose FILE
|
||||
still exists on disk (rename rescues content) from never-committed DB-only
|
||||
rows (delete is the only option). **Effort:** S. **Priority:** P3.
|
||||
- [ ] **P3 — thread source scope into `schema lint --with-db`.** **What:**
|
||||
the stored-type data-plane rules accept `LintOpts.sourceId` (multi-source
|
||||
brains can resolve different packs per source; comparing another source's
|
||||
rows against this manifest yields false alias/undeclared warnings), but
|
||||
neither `src/commands/schema.ts` (`runAllLintRules(pack, { engine })`) nor
|
||||
MCP `schema_lint` passes it — the CLI runs a global scan. Add
|
||||
`--source-id` / honor the worktree pin, and expose `[--json]` in the
|
||||
`jobs stats` usage line while in the area (`src/commands/jobs.ts:309`
|
||||
documents `--queue`/`--cluster-errors` but not the shipped `--json`).
|
||||
Also: the interactive coalesce hint suggests "pass a fresh idempotency
|
||||
key", which `gbrain agent run` has no flag for (raw `jobs submit` does).
|
||||
Surfaced by the v0.46.11.0 post-ship doc review. **Effort:** S.
|
||||
**Priority:** P3.
|
||||
- [ ] **P3 — one-time cross-source clobber audit.** **What:** the
|
||||
pre-guard unscoped-check/scoped-write class could have historically
|
||||
written 'default'-source rows that shadow same-slug rows in other sources.
|
||||
A one-shot integrity probe (`SELECT slug FROM pages GROUP BY slug HAVING
|
||||
count(DISTINCT source_id) > 1` + updated_at ordering heuristics) would
|
||||
surface survivors for review. **Effort:** S. **Priority:** P3.
|
||||
|
||||
## Containment-sprint follow-ups (coverage truth + module peels; plan: ~/.claude/plans/system-instruction-you-are-working-serialized-forest.md)
|
||||
|
||||
- [ ] **P1 — Graduate the diff-coverage gate to blocking (time-boxed 2 weeks from merge).**
|
||||
@@ -381,10 +474,15 @@ Each was explicitly deferred in the pass's CEO/eng/outside-voice reviews.
|
||||
- [ ] **P2 — `jobs submit --max-pending` public flag.** maxPending stays an
|
||||
internal submit option this wave (Codex C4): its semantics exclude
|
||||
delayed/paused/waiting-children rows, and identity is (name, queue, source)
|
||||
so distinct payloads collapse. Decide the public contract (include delayed?
|
||||
explicit scope key?) after the primitive soaks in autopilot, then mirror
|
||||
parseMaxWaitingFlag (clamp [1,100]) + help + flag-registry regen + optional
|
||||
submit_job MCP param. Where: src/commands/jobs.ts, src/core/operations.ts.
|
||||
so distinct payloads collapse. NOTE (five-issue fix wave): the
|
||||
payload-DISTINCT dedupe primitive now exists — admission param-coalescing
|
||||
(`coalesce_params` / minions.coalesce_params.<name>, hash of the full
|
||||
payload incl. owner lane) covers the "identical submits collapse, distinct
|
||||
ones don't" case; --max-pending remains the single-flight-per-scope story.
|
||||
Decide the public contract (include delayed? explicit scope key?) after the
|
||||
primitive soaks in autopilot, then mirror parseMaxWaitingFlag (clamp
|
||||
[1,100]) + help + flag-registry regen + optional submit_job MCP param.
|
||||
Where: src/commands/jobs.ts, src/core/operations.ts.
|
||||
- [ ] **P2 — maxPending at the other single-flight dispatch sites.** The
|
||||
freshness sync submit (src/commands/autopilot.ts freshness loop) and the
|
||||
targeted remediation steps (autopilot.ts targeted-submit loop) still use
|
||||
|
||||
@@ -56,7 +56,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this
|
||||
- `src/core/pool-gauge.ts` — `CheckoutGauge`: approximate in-flight counters at the engine's raw/direct/reserved/tx seams, surfaced via duck-typed `PostgresEngine.getPoolDiagnostics()` (no BrainEngine change, no PGLite stub). HONESTY CONTRACT in the module doc: tagged-template traffic is untracked; consumers must label counts as a subset and never derive waiter/available figures. Fail-open (clamped release, try/finally around sync-throwing runUnsafe). Consumed by db-probe.ts. Pinned by `test/pool-gauge.test.ts`.
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`). Copies the complete source catalog FIRST (`copyMigrationSources` — every `sources` row incl. archived rows and sync/routing metadata, `ON CONFLICT (id) DO UPDATE`, `default` ordered first) so every page write has a valid `pages.source_id` FK parent and the target preserves per-source behavior; pages copy afterward, tracked in the resume manifest by composite `(source_id, slug)` key. Link copy preserves each row's `to_source_id` (falling back to the origin source only for legacy rows without it), and failed-target filtering uses that same target composite key, so cross-source links migrate without being rebound to the origin source. The resume manifest is target-aware: `migrationTargetId(config)` hashes `(engine, locator)` (`database_url` for Postgres, resolved `database_path` for PGLite) and `manifestMatchesTarget` requires `schema_version === 2` plus a matching `target_id` — a legacy engine-only manifest, or one from a DIFFERENT target of the same engine kind, starts fresh instead of skipping "completed" pages the new target never received. Pinned by `test/migrate-engine-resume.test.ts` (manifest identity) + `test/e2e/migrate-engine-sources-postgres.test.ts` (source catalog lands before overlapping-slug pages, PGLite → real Postgres) + `test/e2e/multi-source-bug-class.test.ts` (cross-source links).
|
||||
- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) so a model/dims swap is detectable as stale; `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`), mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle those). `importFromContent`'s tag reconciliation is ADD-ONLY: it only `addTag` (idempotent, ON CONFLICT DO NOTHING). The `tags` table has no provenance column and frontmatter tags are stripped from stored `pages.frontmatter` (markdown.ts:118), so a frontmatter-origin tag can't be distinguished from a DB-enrichment tag (auto-tag / dream synthesize / signal-detector) at re-import — deletion is unsafe (would wipe enrichment under `gbrain reindex --markdown`). Accepted trade-off: removing a tag from frontmatter no longer removes it from the DB on next sync (needs a `tag_source` provenance column). Pinned by `test/reindex-preserve-tags.test.ts` + `test/import-file.test.ts`. identity-based dedup pre-check at `:427-490`. Calls `engine.findDuplicatePage?.(sourceId, {hash, frontmatterId})` (optional `?` so test doubles compile). Posture: SKIP when `frontmatter.id` matches (true external duplicate from overlapping ingest roots), WARN-ALWAYS on content_hash collision with different/missing `frontmatter.id` (templates and daily logs may legitimately share text), FAIL CLOSED on lookup error, bypass via `--force-rechunk`. Soft-deleted pages excluded at the engine layer so tombstones don't block legitimate re-imports under new slugs. Pinned by `test/import-dedup-frontmatter-id.test.ts` (11 cases). `importFromContent` is the narrow waist every ingest path passes through (`gbrain import`, `gbrain sync`, `put_page` MCP, `/ingest` webhook). It runs a three-tier content-quality disposition via `assessContentSanity` from `src/core/content-sanity.ts` BEFORE chunking: (1) high-confidence junk (built-in Cloudflare/CAPTCHA interstitial patterns + operator literals) → QUARANTINE (stamps the `quarantine` frontmatter marker, writes ZERO chunks, hides the page from search) OR REJECT (throw → sync-failure) when `content_sanity.junk_disposition` is `reject`; (2) fuzzy markup-heavy (prose-vs-markup ratio above `content_sanity.max_markup_ratio`, warn-tier byte window, code pages exempt) → `content_flag:markup_heavy` marker (page stays fully searchable, marker rides search results + get_page to warn the agent); (3) oversize → `embed_skip` soft-block via `buildEmbedSkipMarker()` PLUS a `content_flag:oversized` marker, AND deletes any pre-existing chunks in the same transaction so search can't surface stale chunks. Gate-owned markers (`quarantine`, `content_flag`) are STRIPPED from untrusted (remote MCP, `ctx.remote !== false`) frontmatter so a write-scoped client can't hide pages or forge the warning channel; markers are excluded from `content_hash` so a flagged page doesn't re-embed every sync. `gbrain import` honors `errors > 0` for non-zero exit. `classifyErrorCode` in `src/core/sync.ts` recognizes the `PAGE_JUNK_PATTERN` code so sync-failures.jsonl grouping bins these. `extractEntityRefs` (canonical; matches both `[Name](people/slug)` markdown links and Obsidian `[[people/slug|Name]]` wikilinks), `extractPageLinks`, `inferLinkType` heuristics (attended/works_at/invested_in/founded/advises/source/mentions), `parseTimelineEntries`, `isAutoLinkEnabled` config helper. Link candidates match any dir-shaped path (#2576; existence-checked at persist). Used by extract.ts, operations.ts auto-link post-hook, and backlinks.ts. Pinned by `test/import-file-content-sanity.test.ts`.
|
||||
- `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). Exported `pruneDir(name: string): boolean` is the single source of truth for descent-time directory exclusion across walkers — blocks `node_modules` (no leading dot, so naive walkers slipped through and inflated MISSING_OPEN counts via vendor packages), `vendor`/`dist`/`build`/`venv`, dot-prefix dirs, and `*.raw` sidecars — NOT `ops/`, which is ordinary user content (#2404; the bundled daily-task-manager stores `ops/tasks` there); `isSyncable` applies it per path segment, and `walkMarkdownFiles` in `src/commands/extract.ts` + `listTextFiles` in `src/core/cycle/transcript-discovery.ts` consult it BEFORE recursing to save the IO of walking thousands of vendor files (closes #923 + #202). `manageGitignore` worktree discriminator matches the gitdir path segment (`/modules/<name>` = submodule, `/worktrees/<name>` = worktree, per Git's documented layout) so Conductor worktrees (first-class repos) get `.gitignore` management for storage-tiering (closes #889). The sync-failure ledger (failure store, error classifier, the shared bookmark gate, and the doctor severity rule) lives in `src/core/sync-failure-ledger.ts`; `sync.ts` re-exports `classifyErrorCode`, `summarizeFailuresByCode`, `loadSyncFailures`, `unacknowledgedSyncFailures`, `acknowledgeSyncFailures`, `recordSyncFailures`, `decideSyncFailureSeverity`, `applySyncFailureGate`, and the `SyncFailure` type for backward-compatible imports — see its entry below. `isSyncable` factored through private `classifySync(path, opts): SyncableReason | null`; exported companion `unsyncableReason(path, opts)` returns the same tagged reason or null when syncable. `SYNC_SKIP_FILES` is a named export (the four canonical metafile basenames `schema.md`, `index.md`, `log.md`, `README.md`). `SyncableReason` union: `'metafile' | 'strategy' | 'pruned-dir' | 'include-glob-miss' | 'exclude-glob-hit'`. The `commands/sync.ts` cleanup loop guards on `unsyncableReason(path)` being `'metafile'` OR `'pruned-dir'` (#2404) so previously-indexed metafile pages AND deliberately put-created pages under pruned dirs survive every re-sync. Does NOT cover `manifest.deleted` (the upstream filter already strips metafiles). Pinned by `test/sync-isSyncable-shape.test.ts` (15 cases, duality contract) + `test/sync-metafile-skip.serial.test.ts` (3 PGLite cases incl. the renamed `.md → .txt` negative). `pruneDir`: `pruneDir(name, parentDir?)` extended with optional `parentDir`. When provided, additionally rejects directories containing `.git` as a FILE — the git submodule gitfile pattern (regular repos have `.git` as a DIRECTORY; submodules as a file pointing into the parent's `.git/modules/`). Sync + extract walkers thread `parentDir` so the gitfile-as-FILE check fires per descend step. Best-effort: `statSync` failures fall through and treat as a normal dir. Closes the phantom-import bug class where syncing a worktree-with-submodules walked into submodule trees. Pinned by `test/sync-walker-submodule.test.ts`.
|
||||
- `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). Exported `pruneDir(name: string): boolean` is the single source of truth for descent-time directory exclusion across walkers — blocks `node_modules` (no leading dot, so naive walkers slipped through and inflated MISSING_OPEN counts via vendor packages), `vendor`/`dist`/`build`/`venv`, dot-prefix dirs, and `*.raw` sidecars — NOT `ops/`, which is ordinary user content (#2404; the bundled daily-task-manager stores `ops/tasks` there); `isSyncable` applies it per path segment, and `walkMarkdownFiles` in `src/commands/extract.ts` + `listTextFiles` in `src/core/cycle/transcript-discovery.ts` consult it BEFORE recursing to save the IO of walking thousands of vendor files (closes #923 + #202). `manageGitignore` worktree discriminator matches the gitdir path segment (`/modules/<name>` = submodule, `/worktrees/<name>` = worktree, per Git's documented layout) so Conductor worktrees (first-class repos) get `.gitignore` management for storage-tiering (closes #889). The sync-failure ledger (failure store, error classifier, the shared bookmark gate, and the doctor severity rule) lives in `src/core/sync-failure-ledger.ts`; `sync.ts` re-exports `classifyErrorCode`, `summarizeFailuresByCode`, `loadSyncFailures`, `unacknowledgedSyncFailures`, `acknowledgeSyncFailures`, `recordSyncFailures`, `decideSyncFailureSeverity`, `applySyncFailureGate`, and the `SyncFailure` type for backward-compatible imports — see its entry below. `isSyncable` factored through private `classifySync(path, opts): SyncableReason | null`; exported companion `unsyncableReason(path, opts)` returns the same tagged reason or null when syncable. `SYNC_SKIP_FILES` is a named export (the four canonical metafile basenames `schema.md`, `index.md`, `log.md`, `README.md`). `SyncableReason` union: `'metafile' | 'strategy' | 'pruned-dir' | 'include-glob-miss' | 'exclude-glob-hit' | 'malformed-path'`. Malformed filenames are TWO-TIER: `hasMalformedPathSegment` (ADMISSION — control chars on any path; square brackets on `.md`/`.mdx` paths only, so code-strategy lanes keep indexing `app/[id]/page.tsx` framework layouts) vs `isPoisonedPath` (DESTRUCTION — only the injection signature `](` or control chars; sync's row-DELETING lanes gate on this so a bare-bracket markdown row imported by a pre-gate release survives reconcile while its file exists). `sanitizePathForDisplay` scrubs control bytes + caps length before echoing such paths. The `commands/sync.ts` cleanup loop guards on `unsyncableReason(path)` being `'metafile'` OR `'pruned-dir'` (#2404) so previously-indexed metafile pages AND deliberately put-created pages under pruned dirs survive every re-sync. Does NOT cover `manifest.deleted` (the upstream filter already strips metafiles). Pinned by `test/sync-isSyncable-shape.test.ts` (15 cases, duality contract) + `test/sync-metafile-skip.serial.test.ts` (3 PGLite cases incl. the renamed `.md → .txt` negative). `pruneDir`: `pruneDir(name, parentDir?)` extended with optional `parentDir`. When provided, additionally rejects directories containing `.git` as a FILE — the git submodule gitfile pattern (regular repos have `.git` as a DIRECTORY; submodules as a file pointing into the parent's `.git/modules/`). Sync + extract walkers thread `parentDir` so the gitfile-as-FILE check fires per descend step. Best-effort: `statSync` failures fall through and treat as a normal dir. Closes the phantom-import bug class where syncing a worktree-with-submodules walked into submodule trees. Pinned by `test/sync-walker-submodule.test.ts`.
|
||||
- `src/core/sync-failure-ledger.ts` — the bounded auto-skip sync failure ledger (issue #1939; formerly inline "Bug 9" in `sync.ts`). A LEAF module (imports only fs/path/crypto/config) so `sync.ts` can re-export it without a circular dependency. State lives in `~/.gbrain/sync-failures.jsonl`, one JSON object per line, keyed by `(source_id, path)` with a per-key `attempts` count and a 3-state machine: `open` (fresh/blocking) → `auto_skipped` (chronic, still doctor-visible) or `acknowledged` (human resolved via `gbrain sync --skip-failed` from either unresolved state). `classifyErrorCode(errorMsg)` regex classifier with 12 codes (`SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `NESTED_QUOTES`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `INVALID_UTF8`, `STATEMENT_TIMEOUT`, `FILE_TOO_LARGE`, `SYMLINK_NOT_ALLOWED`) plus `UNKNOWN` (also recognizes `PAGE_JUNK_PATTERN` from the content-sanity gate); `summarizeFailuresByCode(failures)` returns sorted `[{code, count}]`; `MISSING_OPEN`/`MISSING_CLOSE`/`EMPTY_FRONTMATTER` regexes match the `markdown.ts` validator strings, `FILE_TOO_LARGE` covers `import-file.ts:199, 352, 401`, `SYMLINK_NOT_ALLOWED` covers `:347`. All mutations run under `withLedgerLock` (cross-process file lock) with an atomic rename write. The auto-skip threshold resolves via `resolveAutoSkipThreshold()` from `GBRAIN_SYNC_AUTOSKIP_AFTER` (default `DEFAULT_AUTOSKIP_AFTER = 3`; `0` disables the valve = pure fail-closed). Two pure decision functions are the unit-test surface: `decideGateAction({fileFailures, sentinels, attemptsByPath, threshold, skipFailed})` returns `hard_block | block | advance | advance_then_autoskip` (sentinels like `<head>` ALWAYS hard-block, even with `--skip-failed`, so a history rewrite can't auto-skip; any FRESH failure with `attempts < threshold` blocks fail-closed; only when ALL failures are chronic does it `advance_then_autoskip`), and `decideSyncFailureSeverity({entries, nowMs, failHours})` returns the `sync_failures` doctor status (`ok` when zero unresolved; `fail` when ≥10 OPEN-blocking or the oldest OPEN failure has blocked the bookmark past `failHours`; otherwise `warn` — `auto_skipped`-only rows stay WARN-visible regardless of count because the bookmark already advanced). `applySyncFailureGate(input)` is the one orchestrator BOTH sync paths (incremental + full/`runImport`) call: it records/clears ledger rows, runs `decideGateAction`, then executes effects in the crash-safe order (advance the bookmark FIRST via the injected `advance()` callback, THEN auto-skip the chronic set) so a crash can never mark a file skipped while leaving sync wedged. `isSkippablePath` rejects `<…>` sentinels. Pinned by `test/sync-failure-ledger.serial.test.ts` + `test/sync-failures.test.ts`.
|
||||
- `src/core/sync-cost-gate.ts` + `sync-git.ts` + `sync-anchor.ts` + `sync-lock.ts` + `sync-reconcile.ts` + `sync-status-report.ts` — six pure-function clusters relocated out of `src/commands/sync.ts` as a pure move; the façade re-exports them, and `facadeExpansion` in `scripts/generate-flag-registry.ts` keeps exactly these six (NOT the pre-existing `sync-*` siblings, which are ordinary deps) on the sync command's flag-scan surface. `sync-cost-gate.ts`: the inline-embed cost gate + token estimation for `gbrain sync`. `sync-git.ts`: git plumbing — invocation building, repo discovery, baseline-commit self-heal, path-containment guards. `sync-anchor.ts`: sync anchor + chunker-version state helpers (source-scoped vs legacy global-config storage). `sync-lock.ts`: the lock layer — typed lock-busy error, the rich busy message, `--break-lock` handling, the partial-result envelope (`performSync` itself stays in the façade). `sync-reconcile.ts`: full-sync reconcile planning (the mass-delete valve and ever-committed gate) + sync deadline/stall resolution. `sync-status-report.ts`: the per-source sync status report backing `gbrain sources status` and the `get_status_snapshot` op.
|
||||
- `src/core/storage.ts` — Pluggable storage interface (S3, Supabase Storage, local).
|
||||
@@ -250,6 +250,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this
|
||||
- `src/core/minions/worker-registry.ts` — live worker registry backing niceness observability. Each running `gbrain jobs work` self-registers `worker-<pid>.json` under `gbrainPath('workers')` (brain-isolated via `GBRAIN_HOME`; entries tagged with `currentBrainId()` so multiple DBs under one home don't cross-report). `registerWorker(info)` is best-effort (never blocks the worker) and returns a cleanup fn the caller wires to BOTH the shutdown `finally` AND `process.on('exit')` (the unhealthy `process.exit(1)` bypasses the awaited finally). `readWorkers(getNice?)` enumerates the dir, drops confirmed-dead pids (`classifyLiveness`: ESRCH = dead/prune, EPERM = alive/keep), applies a pid-reuse start-time guard (`ps -o lstart`, rejects a pid whose process started >5s after the entry was written), and re-measures each live worker's niceness now. Reports the worker's REAL pid, sidestepping the tini-wrapper-pid problem. Pinned by `test/worker-registry.test.ts`.
|
||||
- `src/core/minions/supervisor-pid.ts` — `readSupervisorPid(pidFile) → {pid, running}`: the shared `existsSync → readFileSync → parseInt → process.kill(pid,0)` PID-file + liveness reader extracted from the three copies in `jobs.ts` (supervisor status), `jobs.ts` (stats), and `doctor.ts`. EPERM from the liveness probe counts as running. Pinned by `test/supervisor-pid.test.ts`.
|
||||
- `src/core/minions/handler-timeouts.ts` (#1737, #4145) — per-handler-type defaults for BOTH per-job time knobs, co-located so they can't drift apart unseen (they are different quantities: the budget bounds total runtime, the lease bounds dead-worker reclaim — never derive one from the other). `HANDLER_DEFAULT_TIMEOUT_MS`: 30 min for `subagent`, `subagent_aggregator`, `embed-backfill`, `autopilot-cycle`, `autopilot-global-maintenance`; 10 min for `chronicle_extract` + `facts-absorb`; 60 min for `contextual_reindex_per_chunk`. `HANDLER_DEFAULT_LOCK_DURATION_MS`: 300 s for the long LLM/loop handlers, 120 s for the single-LLM-call handlers, `shell` deliberately absent (fast dead-worker reclaim; verify-before-evict protects it anyway). `defaultTimeoutMsFor` / `defaultLockDurationMsFor` return the mapped default or `null` (short handlers keep the tight null-default wall-clock / the 30 s worker lease). `clampLockDurationMs` + `LOCK_DURATION_MS_MIN/MAX` ([5s, 1h]) is the ONE clamp shared by `queue.add`, the CLI `--lock-duration-ms` flag (`--dry-run` echoes the clamped value), and the MCP `submit_job` param; the same bound is mirrored in SQL at claim time and by the `minion_jobs.lock_duration_ms` range CHECK, so every layer agrees. Layers (explicit value always wins): `MinionQueue.add()` stamps at submit; `MinionQueue.claim()` COALESCEs NULL columns from the maps (durable invariant); migration v128 one-shot backfilled `timeout_ms` with authoring-time snapshot values — do NOT sync v128 when editing the maps (`lock_duration_ms` has no backfill: NULL = worker default = pre-#4145 behavior). Pinned by `test/minions.test.ts` + `test/migrations-v128.test.ts` + `test/migrations-v130.test.ts`.
|
||||
- `src/core/minions/admission.ts` — submit-side queue admission control (the drain-side pool-starvation half is v0.46.1.0's isolation work; claim fairness is a filed TODO, deliberately out of scope). Three primitives resolved per name via `resolveAdmissionPolicy` (config `minions.*` > per-name defaults tables, 60s in-process cache, fail-open with a once-per-process stderr warn; env kill-switch `GBRAIN_MINIONS_ADMISSION=0` disables all three): PARAM-COALESCING (`PARAM_COALESCE_DEFAULT` — on for `subagent`; `computeParamHash` = sha256 of stable-stringified payload excluding only `__param_hash` itself — `__owner_client_id` is deliberately INCLUDED so owner lanes never cross; parentless + waiting-only + age-bounded to ttl/2), WAITING-TTL (`WAITING_TTL_DEFAULT_HOURS` — 48h for `subagent`; swept by `MinionQueue.handleWaitingTTL` through `cancelJobs(ids, {reason, rootStatuses:['waiting']})` so descendants cancel, child_done lands, aggregator parents resolve, and the reason stamps ROOT ids only; ≤500/tick oldest-first; warn-before-act is `runWaitingTtlTick` — first tick counts affected + stamps `TTL_NOTICE_SHOWN_KEY` with an ISO timestamp, sweeping starts only after `ttlNoticeGraceMs()` (1h default, env `GBRAIN_MINIONS_TTL_NOTICE_GRACE_MS`) elapses; `gbrain upgrade` prints the same one-shot notice and starts the same clock; legacy `'true'` flag values sweep immediately), and NAME-GLOBAL QUOTA (`QUOTA_MAX_WAITING_DEFAULT` EMPTY by operator decision — activates only via `minions.quota_max_waiting.<name>`; counts the name across ALL queues so per-run `dream-inline-*` fanout queues can't dodge it, EXACT under concurrency via a `minion_quota:<name>` advisory xact lock taken only when a quota is configured; throws typed `QueueQuotaExceededError`, checked everywhere via `isQueueQuotaExceededError` — dream submitters record a phase skip, synthesize rolls back the current transcript's fresh chunks, agent fanout cancels the whole tree, `submit_agent` maps to a structured `rate_limited` OperationError). `TTL_REASON_PREFIX` is the single source for the sweep's error_text prefix and the stats/doctor LIKE patterns; `safeConfigSegment` gates untrusted job names out of copy-pasteable config hints. Alerting rides `getStats` (`drained_completed/failed/dead/cancelled` keyed on finished_at + `waiting_now` + `oldest_waiting_minutes`), the `jobs stats` DIVERGENT-QUEUE / waiting-TTL screams (`GBRAIN_QUEUE_DIVERGENCE_RATIO`=2, `GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING`=50; divergence compares intake vs COMPLETED so TTL-cancel storms can't masquerade as throughput) + `--json`, and doctor `checkQueueHealth`. Pinned by `test/minions-admission.test.ts` + `test/jobs-stats-divergence.serial.test.ts`.
|
||||
- `src/core/minions/types.ts` — `MinionJobInput` + `MinionJobStatus` + handler context types. `MinionJobInput.max_stalled` is optional; omitted values let the schema DEFAULT (5) kick in, provided values are clamped to `[1, 100]`.
|
||||
- `src/core/minions/protected-names.ts` — side-effect-free constant module exporting `PROTECTED_JOB_NAMES` + `isProtectedJobName()`. Kept pure so queue core can import without loading handler modules. `PROTECTED_JOB_NAMES` includes `synthesize`, `patterns`, `consolidate`. These phases internally submit `subagent` children with `allowProtectedSubmit=true` and can spend Anthropic credits. Only trusted local callers (CLI, autopilot, `doctor --remediate`) can submit them; MCP requests are rejected by `submit_job`'s protected-name guard.
|
||||
- `src/core/minions/handlers/shell.ts` — `shell` job handler. Spawns `/bin/sh -c cmd` (absolute path, PATH-override-safe) or `argv[0] argv[1..]` (no shell). Env allowlist `PATH, HOME, USER, LANG, TZ, NODE_ENV` + caller `env:` overrides + `inherit:`-resolved keys. UTF-8-safe stdout/stderr tail via `string_decoder.StringDecoder`. Abort (either `ctx.signal` or `ctx.shutdownSignal`) fires SIGTERM → 5s grace → SIGKILL on child. Requires `GBRAIN_ALLOW_SHELL_JOBS=1` on worker (gated by `registerBuiltinHandlers`). `ShellJobParams.inherit?: string[]` is a free-form list of snake_case config-key names; the worker resolves each via `loadConfig()` and injects the value under the derived env key (`database_url` → `GBRAIN_DATABASE_URL`; else uppercased). Names persist in `minion_jobs.data` (and the shell-audit JSONL); values never do. The canonical validator `validateShellJobParams` (sibling `shell-validate.ts`) runs PRE-ENQUEUE in both submit surfaces — `gbrain jobs submit shell` (jobs.ts:271) AND the `submit_job` op for `name='shell'` (operations.ts:2085); the handler-entry re-validation here is defense-in-depth (closes the bug class where validation ran AFTER `queue.add()` persisted the row). The validator does NOT police which config keys the agent inherits — same-uid trust model treats the agent as a peer of the worker.
|
||||
@@ -352,9 +353,10 @@ per-release `**vX.Y.Z:**` narration — CI enforces this
|
||||
- `docs/progress-events.md` — Canonical JSON event schema reference. Additive only.
|
||||
- `src/core/markdown.ts` — Frontmatter parsing + body splitter. `coerceFrontmatterString(v)` coerces a non-string `title`/`slug`/`type` to a deterministic string at parse time so a YAML-typed value never reaches `.toLowerCase()` and throws (the #1939 wedge: `title: 2024-06-01` parsed as a `Date`, `title: 1458` as a number, and the throw blocked the sync bookmark from advancing); a `Date` becomes its UTC ISO date (`2024-06-01`, machine-independent and matching the on-disk token, unlike `String(date)`), `null`/`undefined` become `''`, everything else uses `String()`. `splitBody` requires an explicit timeline sentinel (`<!-- timeline -->`, `--- timeline ---`, or `---` immediately before `## Timeline`/`## History`). Plain `---` in body text is a markdown horizontal rule, not a separator. `inferType` auto-types `/wiki/analysis/` → analysis, `/wiki/guides/` → guide, `/wiki/hardware/` → hardware, `/wiki/architecture/` → architecture, `/writing/` → writing (plus existing people/companies/deals/etc heuristics).
|
||||
- `scripts/check-jsonb-pattern.sh` — CI grep guard. Fails the build if anyone reintroduces (a) the `${JSON.stringify(x)}::jsonb` interpolation pattern (postgres.js v3 double-encodes it), or (b) `max_stalled INTEGER NOT NULL DEFAULT 1` in any schema source file (must be DEFAULT 5 to preserve SIGKILL-rescue). It also invokes `scripts/check-jsonb-params.mjs` and propagates its exit code. Wired into `bun test`.
|
||||
- `scripts/check-getpage-scoped-write.mjs` — CI scanner for the unscoped-check/scoped-write source-isolation bug class: flags any non-test src file containing BOTH a `getPage(` call with no second argument (or the `X ? {sourceId} : undefined` any-source-when-unset ternary) AND a write-path call (`putPage(`/`importFromContent(`/`importFromFile(`). Fix pattern: `getPage(slug, { sourceId: x ?? 'default' })` (mirror the write's schema default); opt-out marker `gbrain-allow-unscoped-getpage: <reason>` for documented read-only first-match sites (span, preceding lines, or trailing same-line comment). Grandfathered allowlist is EMPTY. Comment/string-aware balanced-paren span walker (same skeleton as check-jsonb-params.mjs); argv-overridable roots; wired as `check:getpage-scope` in verify CHECKS + guards-manifest + `test/fixtures/guards/check-getpage-scoped-write.mjs/{bad,good}/` + `test/check-getpage-scoped-write.test.ts`.
|
||||
- `scripts/check-jsonb-params.mjs` — AST-lite CI guard for the POSITIONAL jsonb double-encode form the template grep above misses: an `executeRaw`/`executeRawDirect`/`.unsafe()` call whose balanced arg span binds `JSON.stringify(x)` into a bare `$N::jsonb` cast (the #2339 class). Walks each call's balanced span respecting strings/templates/comments, handles generic-typed calls (`executeRaw<T>(`), and allows the sanctioned forms (`$N::text::jsonb`, `$N::text[]`, `executeRawJsonb`, `sql.json`, an inline `jsonb-guard-ok` comment). PGLite's native `db.query` is deliberately not scanned (it parses text→jsonb, so the bug can't occur there). Heuristic by design (whole-span correlation; can't see a `JSON.stringify` assigned to a variable before the call) — the real backstop is the DATABASE_URL-gated e2e parity tests. Scan roots overridable via argv for its self-test (`test/check-jsonb-params.test.ts`).
|
||||
- `scripts/check-source-id-projection.sh` — CI grep guard for the multi-source bug class. Greps `src/core/postgres-engine.ts` + `src/core/pglite-engine.ts` for `SELECT.*FROM pages` projections matching the `rowToPage` feeder shape (id + slug + type + title) and fails if `source_id` is missing. `Page.source_id` is required at the type level; a projection dropping the column produces `Page` rows with `source_id: undefined` while TypeScript's `: string` lies about it. Wired into `bun run verify`.
|
||||
- `scripts/guards-manifest.tsv` + `scripts/guard-self-test.sh` — THE single registry of `scripts/check-*` CI guards (48 guards) and its self-test harness. Every guard is classified `scanner` (greps/parses repo sources — must eventually carry fixtures), `buildfresh`, or `repostate` (exempt-with-reason, not fixture-tested). `guard-self-test.sh` (`bun run check:guard-self-test`, wired into `bun run verify`) runs each `selftest=yes` scanner against known-bad (must exit non-zero) and known-good (must pass) fixture trees under `test/fixtures/guards/<guard>/{bad,good}/` via the `GBRAIN_GUARD_ROOT` env seam, and fails the build when a new `scripts/check-*` script is missing from the manifest — so a guard whose pattern rots into a permanently-green no-op fails CI instead of masquerading as coverage. The manifest replaces the second hand-synced REGISTRY copy (package.json's `check:all` chain — deleted); it registers and classifies guards but does not itself schedule them — `run-verify-parallel.sh`'s `CHECKS` array remains the execution list, and a registered guard is not automatically wired into verify. New guard = new manifest row (+ fixtures if scanner) + a `CHECKS` entry if it should gate pushes.
|
||||
- `scripts/guards-manifest.tsv` + `scripts/guard-self-test.sh` — THE single registry of `scripts/check-*` CI guards (52 guards) and its self-test harness. Every guard is classified `scanner` (greps/parses repo sources — must eventually carry fixtures), `buildfresh`, or `repostate` (exempt-with-reason, not fixture-tested). `guard-self-test.sh` (`bun run check:guard-self-test`, wired into `bun run verify`) runs each `selftest=yes` scanner against known-bad (must exit non-zero) and known-good (must pass) fixture trees under `test/fixtures/guards/<guard>/{bad,good}/` via the `GBRAIN_GUARD_ROOT` env seam, and fails the build when a new `scripts/check-*` script is missing from the manifest — so a guard whose pattern rots into a permanently-green no-op fails CI instead of masquerading as coverage. The manifest replaces the second hand-synced REGISTRY copy (package.json's `check:all` chain — deleted); it registers and classifies guards but does not itself schedule them — `run-verify-parallel.sh`'s `CHECKS` array remains the execution list, and a registered guard is not automatically wired into verify. New guard = new manifest row (+ fixtures if scanner) + a `CHECKS` entry if it should gate pushes.
|
||||
- `scripts/merge-lcov.ts` + `scripts/coverage-diff-gate.ts` + `scripts/coverage-baseline-gate.ts` + `scripts/update-coverage-baseline.ts` + `scripts/render-coverage-summary.ts` + `scripts/coverage-gate-exemptions.txt` + `scripts/coverage-baseline.json` — the coverage measurement + gating cluster; the operating guide is docs/TESTING.md "Coverage lanes and gates". `merge-lcov.ts` walks artifact dirs for `lcov.info` + `lane-manifest.json`, sums DA hits per file:line, normalizes paths repo-relative, and emits a merged lcov + summary JSON (src-only totals/per-dir/per-file, the `lineHits` extension the diff gate consumes, and never-loaded src files as count + sorted list — deliberately never a percentage, since physical lines ≠ executable lines); `--manifest-expect` pins the lane set, and a missing/incomplete lane or a `shard` lane with `lcovCount != 1` (the xargs-batching tripwire) marks the summary `degraded: true` — still exit 0 (degraded is data, and both gates go report-only on it). `coverage-diff-gate.ts` gates added/changed gate-scoped lines (non-test, non-generated `src/**.ts`) at ≥80% covered plus zero changed-but-never-loaded files; report-only unless `COVERAGE_GATE_ENFORCE=1`; a `[coverage-exempt: reason]` commit trailer passes with a loud warning; `coverage-gate-exemptions.txt` rows (exact path or trailing-`/` prefix; SHRINK-ONLY — additions need a graduation review) are excluded from the gate but still reported (`[e2e-exempt]` / `[subprocess-undercount]`); exit contract: 0 = pass or report-only, 1 = fail while enforcing, 2 = infrastructure error (never conflated with a coverage verdict). `coverage-baseline-gate.ts` reads the baseline via `git show origin/master:scripts/coverage-baseline.json` (never the working tree, so a PR can't weaken its own bar) and compares corpus-matched sections only (`--corpus prCorpus|fullCorpus`), failing on >0.5pp global or >1.0pp per-dir drops; `provisional: true` in the baseline (the current state — both corpus sections unseeded) keeps it report-only regardless of enforcement; `update-coverage-baseline.ts` writes the working-tree baseline (per-file detail limited to the committed `watchlist`) and `--promote` flips `provisional: false`. `render-coverage-summary.ts` renders the summary JSON as markdown on stdout for `$GITHUB_STEP_SUMMARY`, including the behavioral-vs-structural counts from `scripts/structural-suites.tsv`. Wiring: 13 PR-corpus lanes in test.yml (10 matrix shards + serial + the two dedicated slow jobs) upload `coverage-*` artifacts and the advisory `coverage-report` job merges + renders + runs both gates report-only (deliberately absent from `test-status`/`cache-write` until graduation); schedule-only `coverage-full-{unit,serial,slow,e2e}` + `coverage-full-report` in e2e.yml produce the self-contained nightly fullCorpus number (full e2e glob included) and the `coverage-full-merged` trend artifact. Collection is `COVERAGE_DIR`-opt-in in `test-shard.sh`/`run-serial-tests.sh`/`run-e2e.sh` — unique coverage dir per bun process (a reused dir overwrites `lcov.info`), lane manifest written only on a green run, `run-e2e.sh` requires an ABSOLUTE `COVERAGE_DIR` and honors `E2E_FILE_TIMEOUT_SECS` (both deliberately non-`GBRAIN_`-prefixed to survive the hermetic env scrub). Bun/JSC emits line records only (function coverage is informational) and no subprocess coverage, so `src/cli.ts` undercounts. Pinned by `test/scripts/merge-lcov.test.ts`, `test/scripts/coverage-diff-gate.test.ts`, `test/scripts/render-coverage-summary.test.ts`.
|
||||
- `scripts/check-module-size.sh` + `scripts/module-size-limits.tsv` — the module-size ratchet (`bun run check:module-size`, wired into `bun run verify`). The TSV commits a per-file `wc -l` ceiling (`path max_lines policy note`); four rules, all violations reported before a single exit 1: a file above its ceiling fails (raise a ceiling only as a conscious TSV edit); a ceiling more than 50 lines above the measured size fails (stale slack after a shrink — lower it so the ratchet holds); a TSV row whose path no longer exists fails (remove the row); an unlisted `src/**/*.ts` (excluding `*.generated.ts`/`*.test.ts`) above the 1500-line new-file cap fails (split it or add a row). Policy `region-exempt` (only `src/core/migrate.ts`) counts lines OUTSIDE the append-only `export const MIGRATIONS = [` … `];` region, so the migrations array grows freely while the surrounding runner logic stays ratcheted. Self-test seams: `GBRAIN_GUARD_ROOT`, `GBRAIN_MODULE_SIZE_SLACK`, `GBRAIN_MODULE_SIZE_NEWFILE_CAP`.
|
||||
- `scripts/classify-tests.ts` + `scripts/structural-suites.tsv` — suite-level behavioral-vs-structural test classification (the intent axis described in docs/TESTING.md "File taxonomy"). Content-based detectors — repo-anchored `readFileSync`/`Bun.file` readers, exec-scan grep windows over `src|scripts|docs`, and the `doctorSource()`/`doctorFileSource()` helpers — mark a suite STRUCTURAL when its assertions read repo source/doc text rather than executing product code; tmpdir-anchored reads don't count, and files with detectors but no attributable suite land in an `unknown` bucket emitted as comment rows (surfaced, never silently dropped). Modes: bare = rewrite the TSV; `--check` = byte-for-byte regenerate-and-diff freshness (wired as `bun run check:structural-manifest` in `bun run verify` via `scripts/check-structural-manifest.sh`); `--summary` = counts only. Fix misclassifications in the detector list, never by hand-editing the TSV. `render-coverage-summary.ts` consumes the TSV for the behavioral-vs-structural line in the CI coverage report.
|
||||
@@ -499,12 +501,14 @@ credit `@garrytan-agents`): six foundation modules + a mutate skeleton +
|
||||
stats/sync data plane + CLI verbs + MCP ops + a first-class agent skill.
|
||||
|
||||
Key files (v0.40.7.0 additions):
|
||||
- `src/core/atomic-write.ts` — the shared atomic file writer for brain-repo markdown writers: unique tmp sibling (`.tmp.<pid>.<rand>`) → write loop until every byte lands (writeSync may legally short-write under disk pressure; a silent short write could atomically install truncated content) → fsync → close → optional `verify(onDiskBytes)` callback (throw = abort, tmp removed, target untouched) → mode-preserving atomic rename → best-effort parent-directory fsync (rename durability). Born from the backlinks frontmatter-corruption incident; currently consumed by `src/commands/backlinks.ts` (which verifies with `parseMarkdown({validate:true})` before the rename). Rename prevents torn writes, NOT lost updates — read-modify-write callers pair it with `withPageLock` (backlinks does). Migrating the older per-module copies (skillopt/apply-edits, write-through, lint) is a filed TODO.
|
||||
- `src/core/schema-pack/pack-lock.ts` — Atomic `O_CREAT|O_EXCL` per-pack lock. DELIBERATELY NOT the `existsSync + writeFileSync` TOCTOU shape from `src/core/page-lock.ts`. Default 60s TTL, refresh every 10s while `withPackLock(fn)` runs, `--force` semantics = "steal stale lock" NOT "skip locking." Lock path per-pack so two packs never block each other.
|
||||
- `src/core/schema-pack/mutate-audit.ts` — ISO-week JSONL at `~/.gbrain/audit/schema-mutations-YYYY-Www.jsonl`. Privacy-redacted: type names → sha8, prefixes → first slug segment only, matches `candidate-audit.ts` privacy posture. Logs BOTH success AND failure events so the `schema_pack_writability` doctor check has signal. `summarizeMutations()` is the cross-surface parity primitive.
|
||||
- `src/core/schema-pack/registry.ts` — `resolvePack` walks the `extends` chain (depth cap via `EXTENDS_DEPTH_WARN` / `EXTENDS_DEPTH_HARD_CAP`), RETAINS each ancestor manifest, materializes `borrow_from`, and composes all of it into `resolved.manifest` through `mergeInheritedManifest`. Every downstream consumer reads `resolved.manifest`, so doing the merge here is what makes inheritance visible without per-consumer wiring. `borrow_from` is selective (only the named `types` / `link_types`, and only from the target's OWN declarations), non-transitive, and fail-closed — a missing target throws `UnknownPackError` via `loadByName`, matching the extends path; an omitted category borrows none of it. The alias graph + closure hash are computed on the MERGED manifest, so a cross-pack alias cycle surfaces as `AliasCycleError` at resolve. `manifest_sha8` / `packIdentity` stay the CHILD's own bytes — a parent edit does not move the child's identity, so the invalidation path is what keeps a child honest. `invalidatePackCache(name?)` walks the extends-chain reverse-graph (editing a parent pack must not leave children stale). `tryCachedPack(name)` TTL-gated fast path: inside `STAT_TTL_MS` (default 1000ms, env `GBRAIN_PACK_STAT_TTL_MS`) returns cached without statting; outside the window it stats every TRACKED file — the extends chain PLUS every borrowed pack — and cascade-invalidates on mtime change (cross-process detection), so editing a borrowed pack invalidates its borrowers. Pinned by `test/schema-pack-registry.test.ts` + `test/schema-pack-merge.test.ts`.
|
||||
- `src/core/schema-pack/merge.ts` — the pure child-wins composition helper behind `resolvePack`. `mergeInheritedManifest(ancestorsBaseFirst, child, borrowed)` returns the fully-composed manifest; precedence is child → borrowed → nearest parent … → base. SIX ingest/query-shaping fields inherit: `page_types`, `link_types`, `frontmatter_links`, `enrichable_types`, `filing_rules`, `takes_kinds`. `phases` + `calibration_domains` are DELIBERATELY child-only — they gate real cycle execution (`cycle.ts` `packDeclaresPhase`), so inheriting them would silently run phases a pack never declared; `mapping_rules`, `migration_from`, `extends`, `borrow_from`, and the identity fields are child-only too (all ride the `...child` spread). `mergePageTypes` carries the ordering contract `inferTypeFromPack` depends on (first-`path_prefix`-match-wins, array order): the BASE (root, `extends: null`) pack is the ordered foundation/tail; an override of a base type keeps the base POSITION (`Map.set` updates the value, keeps insertion order) so base's curated priority survives; a genuinely-new type from ANY non-base layer — child, borrowed, or a middle pack — is PREPENDED nearest-first, so a more-derived prefix wins regardless of chain depth. `mergeByKey` keeps the first occurrence per key walking highest-precedence-first (the order-insensitive keyed fields); `frontmatter_links` keys on `page_type\x00link_type` — a NUL, not a space, because both are unconstrained strings and a space-join would collide `{"a b","c"}` with `{"a","b c"}`. `mergeUnion` backs `takes_kinds`: UNION not replace, because the Zod default makes an omitted field indistinguishable from an explicit one — so a child can ADD kinds but CANNOT narrow below base ∪ parent. Pure + deterministic: no disk, no engine. Pinned by `test/schema-pack-merge.test.ts`.
|
||||
- `src/core/schema-pack/best-effort.ts` — `loadActivePackBestEffort(ctx)` returns `ResolvedPack | null`. Single source of truth for the T1.5 wiring sites. `null` means EMPTY FILTER (NOT hardcoded defaults — closes the silent-violation bug class).
|
||||
- `src/core/schema-pack/lint-rules.ts` — 12 pure rule functions. `withMutation`'s pre-write validation gate composes the 10 file-plane rules; the 2 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`) need an engine. Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. New file-plane rule `link_regex_catastrophic_backtrack` — advisory ReDoS pre-screen flagging the classic nested-quantifier shapes (`(a+)+`, `(a*)*`, `(a+)*`, `(\w+)+`) in a link_type's `inference.regex` via `NESTED_QUANTIFIER_RE`. WARNING not error: a hard reject would disable the whole pack on upgrade (pages fall back to legacy typing). The runtime input-length cap in `redos-guard.ts` is the actual safety net; this rule tells the pack author to fix the pattern.
|
||||
- `src/core/schema-pack/type-usage.ts` — stored-type classifier behind the alias-footgun visibility surfaces: `classifyStoredType(type, pack)` → canonical | alias_of (with the canonical type + `path_prefixes[0]` filing directory) | undeclared, over a STRUCTURAL pack shape so import-file's thin `activePack` and the full manifest both satisfy it. `sanitizeTypeForDisplay` strips control chars + caps length (type strings come from frontmatter and get echoed into terminals); `renderTypeWarningSummary` renders the once-per-type-per-run lines. Consumers: `importFromContent` (advisory `ImportResult.type_warning` at the typeExplicit site — the type is still stored literally, zero filing change), sync/import summary aggregation (+ `SyncResult.type_warnings` so worker-driven syncs surface counts in job results), the `stored_type_is_alias`/`stored_type_undeclared` data-plane lint rules, all gated by config `schema.type_warnings` (default on; lint rules always active). Pinned by `test/schema-type-usage.test.ts`.
|
||||
- `src/core/schema-pack/lint-rules.ts` — 14 pure rule functions. `withMutation`'s pre-write validation gate composes the 10 file-plane rules; the 4 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`, `stored_type_is_alias`, `stored_type_undeclared`) need an engine (CLI `--with-db`; the stored-type pair accepts `LintOpts.sourceId` scoping, not yet threaded from the CLI). Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. New file-plane rule `link_regex_catastrophic_backtrack` — advisory ReDoS pre-screen flagging the classic nested-quantifier shapes (`(a+)+`, `(a*)*`, `(a+)*`, `(\w+)+`) in a link_type's `inference.regex` via `NESTED_QUANTIFIER_RE`. WARNING not error: a hard reject would disable the whole pack on upgrade (pages fall back to legacy typing). The runtime input-length cap in `redos-guard.ts` is the actual safety net; this rule tells the pack author to fix the pattern.
|
||||
- `src/core/schema-pack/redos-guard.ts` + `src/core/schema-pack/link-inference.ts` — ReDoS hardening for pack inference regexes. `redos-guard.ts` adds `MAX_REGEX_INPUT_CHARS` (default 64_000, env `GBRAIN_MAX_REGEX_INPUT_CHARS`) — a hard input-length cap, the real runtime safety net (catastrophic backtracking needs a long input; a link-extraction `context` is normally a sentence or short paragraph). Over the cap, `runRegexBounded` throws the tagged `RegexInputTooLargeError` and the regex is skipped (degrade-to-mentions) without entering the `node:vm`. `link-inference.ts:inferLinkTypeFromPack` no-budget branch (test contexts) now routes through `runRegexBounded` so the input-length cap + per-regex vm timeout (`PER_REGEX_TIMEOUT_MS = 50`) apply on every path (previously this branch ran `new RegExp(pattern).test(context)` unbounded — the one ReDoS hole with no timeout). Defensive hardening + diagnostics; the deterministic ~3100-file sync-wedge root cause remains open. Pinned by `test/redos-hardening.test.ts` + `test/schema-pack-lint-rules.test.ts`.
|
||||
- `src/core/schema-pack/query-cache-invalidator.ts` — `invalidateQueryCache(engine, sourceId?)` DELETEs query_cache rows so cached search results bound to old page types don't survive a schema mutation.
|
||||
- `src/core/schema-pack/mutate.ts` — 8-step `withMutation` skeleton (bundled-guard → lock → read → mutator → validate → atomic write → audit → invalidate) backs the 11 single-mutation primitives: `addTypeToPack`, `removeTypeFromPack` (with reference check), `updateTypeOnPack`, `addAliasToType`, `removeAliasFromType`, `addPrefixToType`, `removePrefixFromType`, `addLinkTypeToPack`, `removeLinkTypeFromPack`, `setExtractableOnType`, `setExpertRoutingOnType`. Each primitive's business-rule validation + transform is factored into a `build*Mutator(...)` pure `(manifest) => manifest` function shared with `applyMutationsAtomic` (the `schema_apply_mutations` batch entry point) so single-call and batched mutations can never validate differently. `applyMutationsAtomic` locks + reads the pack file ONCE, applies + lint-validates every mutation in the batch against an in-memory manifest, and calls `writePackManifest` at MOST ONCE — only after the whole batch checks out — so a batch that fails partway leaves the pack file byte-identical to its pre-batch state. Atomic single write via `.tmp + fsync + rename` — the pack file on disk is NEVER partial, for either a single mutation or a batch. Inline minimal JSON→YAML emitter so YAML packs stay YAML (does NOT preserve comments — pin pack.json if you care about layout).
|
||||
|
||||
@@ -69,7 +69,7 @@ gbrain schema fork <a> <b> # copy + rename a pack (experimental)
|
||||
gbrain schema edit <name> # surface the pack path (experimental)
|
||||
gbrain schema diff <a> <b> # set-diff two packs (experimental)
|
||||
gbrain schema graph # ASCII type listing (experimental)
|
||||
gbrain schema lint # flag duplicates + missing prefixes
|
||||
gbrain schema lint [--with-db] # duplicates + missing prefixes; --with-db adds data-plane rules
|
||||
gbrain schema explain <type> # plain-English type description (experimental)
|
||||
gbrain schema downgrade --to <p> # restore previous pack (recovery)
|
||||
gbrain schema usage --since 30d # per-verb invocation counts (telemetry)
|
||||
@@ -79,6 +79,18 @@ The verbs marked `experimental` are demand-gated: usage is tracked via the
|
||||
schema-events audit (`gbrain schema usage`), which informs whether
|
||||
rarely-used verbs get deprecated.
|
||||
|
||||
With `--with-db`, `schema lint` also runs two data-plane rules over the
|
||||
stored corpus: `stored_type_is_alias` (a page's explicit type is an alias —
|
||||
the canonical type and its filing directory are named) and
|
||||
`stored_type_undeclared` (the type isn't in the active pack at all). The
|
||||
rule layer accepts a per-source scope (`LintOpts.sourceId` — multi-source
|
||||
brains can resolve different packs per source), though the CLI currently
|
||||
runs a global scan. The same classification warns once per type per run at
|
||||
sync/import so alias types stop filing into unexpected directories
|
||||
silently; silence the ingest warnings with
|
||||
`gbrain config set schema.type_warnings false` (the `--with-db` lint rules
|
||||
are unaffected).
|
||||
|
||||
## Resolution chain (7 tiers)
|
||||
|
||||
When the engine decides "which pack is active for this query?", it walks
|
||||
|
||||
@@ -12,9 +12,13 @@ The persistent worker can die silently from:
|
||||
- Bun process crashes with no automatic restart.
|
||||
- Internal event-loop death (PID alive, worker loop stopped).
|
||||
|
||||
When the worker dies, submitted jobs sit in `waiting` forever. The
|
||||
canonical answer is `gbrain jobs supervisor` — a first-class CLI that
|
||||
spawns `gbrain jobs work` as a child and auto-restarts it on crash.
|
||||
When the worker dies, submitted jobs sit in `waiting` — indefinitely for
|
||||
most types; types with a waiting-TTL (`subagent` defaults to 48h, see the
|
||||
[queue operations runbook](queue-operations-runbook.md)) are eventually
|
||||
cancelled with an auditable reason rather than queueing forever. Either
|
||||
way the work doesn't happen. The canonical answer is
|
||||
`gbrain jobs supervisor` — a first-class CLI that spawns `gbrain jobs work`
|
||||
as a child and auto-restarts it on crash.
|
||||
|
||||
## Worker supervision
|
||||
|
||||
|
||||
@@ -55,6 +55,45 @@ gbrain jobs supervisor stop && gbrain jobs supervisor start --detach --json
|
||||
gbrain jobs retry <id>
|
||||
```
|
||||
|
||||
## The backlog grows structurally (DIVERGENT QUEUE)
|
||||
|
||||
A different failure from a wedge: the worker is draining fine, but one job
|
||||
type's intake structurally exceeds its completions, so the waiting pile
|
||||
grows forever. Since v0.46.11.0 the queue has admission control and the
|
||||
signal is loud:
|
||||
|
||||
```bash
|
||||
gbrain jobs stats # Drained/Waiting columns + a DIVERGENT QUEUE
|
||||
# scream per offending type (also in --json)
|
||||
gbrain doctor --json | jq '.checks[] | select(.name == "queue_health")'
|
||||
# same findings for cron topologies
|
||||
```
|
||||
|
||||
The scream fires when a type's 24h intake exceeds `GBRAIN_QUEUE_DIVERGENCE_RATIO`
|
||||
(default 2) × its 24h completions AND more than
|
||||
`GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING` (default 50) jobs are waiting.
|
||||
Cancellations — including the waiting-TTL sweep — are deliberately not
|
||||
counted as drain: outflow is not work.
|
||||
|
||||
What's already protecting you, and the knobs:
|
||||
|
||||
- **Param-coalescing** (default on for `subagent`): identical parentless
|
||||
submits — same owner lane, payload, and execution options — coalesce onto
|
||||
the existing waiting job instead of stacking. Per-name toggle:
|
||||
`minions.coalesce_params.<name>`.
|
||||
- **Waiting-TTL** (default 48h for `subagent`): jobs still waiting past the
|
||||
TTL are cancelled with an auditable reason instead of queueing forever.
|
||||
Tune or disable: `gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`.
|
||||
The first sweep never fires cold — a one-time notice prints with the
|
||||
affected-job count, then a one-hour grace window holds before the first
|
||||
cancellation.
|
||||
- **Waiting quota** (opt-in, off by default): a hard cap on a type's waiting
|
||||
count, name-global across queues, exact under concurrent submitters. New
|
||||
submits past the cap are rejected with a structured, retryable error.
|
||||
Opt in: `gbrain config set minions.quota_max_waiting.<name> <n>`.
|
||||
- **Kill-switch**: `GBRAIN_MINIONS_ADMISSION=0` disables all three at once
|
||||
(incident escape hatch, no DB needed).
|
||||
|
||||
## Triage commands
|
||||
|
||||
```bash
|
||||
@@ -106,6 +145,16 @@ gbrain jobs smoke --wedge-rescue
|
||||
drain them. Set `--max-waiting N` on the submission or on the programmatic
|
||||
`queue.add()` call. If you want a taller pile, raise the threshold via
|
||||
`GBRAIN_QUEUE_WAITING_THRESHOLD=50 gbrain doctor`.
|
||||
- **divergent queue** — A type's 24h intake structurally exceeds its 24h
|
||||
completions while a real backlog waits (same thresholds as the
|
||||
`jobs stats` scream, so the two surfaces agree). The finding names the
|
||||
type and prints the exact `minions.quota_max_waiting.<name>` command to
|
||||
cap admission. See "The backlog grows structurally" above.
|
||||
- **waiting-TTL cancellations** — The admission sweep cancelled queued work
|
||||
that expired unclaimed in the last 24h. That's operating as designed, but
|
||||
it means the divergence is being shredded, not worked — intake still
|
||||
exceeds drain. Tune with `gbrain config set
|
||||
minions.ttl_waiting_hours.<name> <hours|0>`.
|
||||
|
||||
## Lock-renewal: reading an eviction, and the knobs
|
||||
|
||||
|
||||
@@ -151,6 +151,9 @@ Stable phase names shipped in v0.15.2:
|
||||
writer adds chunks mid-run)
|
||||
- `repair_jsonb.run`, `repair_jsonb.<table>.<column>`
|
||||
- `backlinks.scan`
|
||||
- `backlinks.fix` — heartbeat-only (no total): the fix loop runs per-file
|
||||
locking + parse-validation + atomic writes, so agents see forward progress
|
||||
while it works through the gap list
|
||||
- `lint.pages`
|
||||
- `integrity.auto`
|
||||
- `eval.single`, `eval.ab`
|
||||
|
||||
@@ -204,7 +204,7 @@ gbrain schema add-alias researcher person
|
||||
|
||||
Read [`skills/conventions/schema-evolution.md`](../skills/conventions/schema-evolution.md) for the decision tree on when to add types vs aliases vs prefixes. The short version: <20 pages → don't pack-codify; 20-100 → alias on existing type; 100+ → first-class type.
|
||||
|
||||
**Lint your pack before shipping.** The 11-rule lint surface (with the optional `--with-db` flag for DB-aware checks) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
**Lint your pack before shipping.** The 14-rule lint surface (with the optional `--with-db` flag for DB-aware checks, including the stored-type alias/undeclared rules) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
|
||||
```bash
|
||||
gbrain schema lint --with-db
|
||||
|
||||
+7
-2
@@ -213,7 +213,12 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
sites; `ctx.remote !== false` for untrust-unless-explicit-false). Don't default it falsy.
|
||||
- **Source isolation.** Every read-side op routes through `sourceScopeOpts(ctx)`; precedence
|
||||
is federated array (`ctx.auth.allowedSources`) > scalar (`ctx.sourceId`) > nothing. Don't
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak.
|
||||
hand-roll source filtering — a missed thread is a cross-source data leak. Corollary
|
||||
(unscoped-check/scoped-write): `engine.getPage` with no opts matches ANY source while
|
||||
`putPage` defaults to `'default'` — an existence check + write pair must scope the read
|
||||
to the write's source (`getPage(slug, { sourceId: x ?? 'default' })`). Guarded by
|
||||
`scripts/check-getpage-scoped-write.mjs` (opt-out marker
|
||||
`gbrain-allow-unscoped-getpage` for read-only first-match sites).
|
||||
- **JSONB: never `JSON.stringify` into a `::jsonb` cast.** postgres.js double-encodes it (a jsonb
|
||||
string scalar); PGLite hides the bug. This bites BOTH spellings — the template form
|
||||
(`${JSON.stringify(x)}::jsonb`) AND the positional form (`executeRaw(\`…$N::jsonb\`, [JSON.stringify(x)])`,
|
||||
@@ -2936,7 +2941,7 @@ gbrain schema add-alias researcher person
|
||||
|
||||
Read [`skills/conventions/schema-evolution.md`](../skills/conventions/schema-evolution.md) for the decision tree on when to add types vs aliases vs prefixes. The short version: <20 pages → don't pack-codify; 20-100 → alias on existing type; 100+ → first-class type.
|
||||
|
||||
**Lint your pack before shipping.** The 11-rule lint surface (with the optional `--with-db` flag for DB-aware checks) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
**Lint your pack before shipping.** The 14-rule lint surface (with the optional `--with-db` flag for DB-aware checks, including the stored-type alias/undeclared rules) catches dangling references, prefix collisions, and dead-corpus warnings:
|
||||
|
||||
```bash
|
||||
gbrain schema lint --with-db
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.11.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
|
||||
+2
-1
@@ -100,6 +100,7 @@
|
||||
"check:pagetype-exhaustive": "bash scripts/check-pagetype-exhaustive.sh",
|
||||
"check:pg-url-redaction": "bash scripts/check-pg-url-redaction.sh",
|
||||
"check:source-scope-onboard": "bash scripts/check-source-scope-onboard.sh",
|
||||
"check:getpage-scope": "node scripts/check-getpage-scoped-write.mjs",
|
||||
"postinstall": "bun run scripts/postinstall.ts",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin",
|
||||
@@ -167,7 +168,7 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.46.10.0",
|
||||
"version": "0.46.11.0",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.5",
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-plugin-tree-stamp: 0.46.10.0 -->
|
||||
<!-- gbrain-plugin-tree-stamp: 0.46.11.0 -->
|
||||
# gbrain plugin skill tree (generated — do not hand-edit)
|
||||
|
||||
This tree is the curated skill set for the gbrain Codex and Claude Code
|
||||
|
||||
@@ -236,6 +236,18 @@ Queue/priority/retry tuning is not exposed by `gbrain agent run`; submit the
|
||||
raw `subagent` handler via `gbrain jobs submit` (requires CLI trust) if you
|
||||
need those knobs.
|
||||
|
||||
**Admission control (v0.46.11.0).** Identical parentless `subagent` submits
|
||||
(same owner lane, payload, and execution options) coalesce onto the existing
|
||||
waiting job: `gbrain agent run` prints `coalesced` with the matched job id,
|
||||
and the `submit_agent` MCP response carries `coalesced: true`. Treat that as
|
||||
success — monitor the matched id, do NOT resubmit. Jobs still waiting after
|
||||
the TTL (48h default for `subagent`; `minions.ttl_waiting_hours.<name>`)
|
||||
are cancelled with reason prefix `waiting_ttl_expired`. If an operator has
|
||||
configured a waiting quota (`minions.quota_max_waiting.<name>`), a submit
|
||||
past the cap returns a structured, retryable `rate_limited` error — back
|
||||
off and check `gbrain jobs stats` for a `DIVERGENT QUEUE` line before
|
||||
retrying.
|
||||
|
||||
## Phase 2: Monitor
|
||||
|
||||
```
|
||||
@@ -488,6 +500,7 @@ Total tokens so far: 4.3k
|
||||
- Don't spawn a Minion for a single search query (use search tool directly)
|
||||
- Don't fire-and-forget without checking results
|
||||
- Don't spawn > 5 concurrent agents without checking `gbrain jobs stats` first
|
||||
- Don't resubmit when a submit reports `coalesced` — the work is already queued; monitor the matched job id instead
|
||||
- For subagent work, don't use `sessions_spawn` with `runtime: "subagent"` when Minions is available (use `gbrain agent run` instead)
|
||||
- Don't poll `get_job` in a tight loop (use `get_job_progress` for lightweight checks)
|
||||
- Don't run an operation expected to exceed ~2 minutes as a bare background shell — it dies with the session; route through the Durable execution ladder
|
||||
|
||||
@@ -177,8 +177,9 @@ Validate before sync:
|
||||
gbrain schema lint --with-db
|
||||
```
|
||||
|
||||
The `--with-db` flag opts into the 2 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`) that detect
|
||||
The `--with-db` flag opts into the 4 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`,
|
||||
`stored_type_is_alias`, `stored_type_undeclared`) that detect
|
||||
mis-declared types you'd otherwise discover only at runtime.
|
||||
|
||||
### Phase 5 — Sync (backfill existing pages with the new types)
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* CI guard for the unscoped-check/scoped-write source-isolation bug class.
|
||||
*
|
||||
* The trap: `engine.getPage(slug)` with NO opts matches the slug in ANY
|
||||
* source (first row wins), while the paired write (`putPage` /
|
||||
* `importFromContent` / `tx.putPage`) defaults to the 'default' source. A
|
||||
* page that exists only in source B makes the existence check "succeed",
|
||||
* and the write then targets a DIFFERENT row — duplicates, clobbers, or
|
||||
* crashes (this class broke dream cycles for weeks; the writer/slug-registry
|
||||
* variant forced spurious slug disambiguation).
|
||||
*
|
||||
* Heuristic (deliberately file-scoped, same posture as
|
||||
* check-source-scope-onboard.sh): flag any non-test source file that contains
|
||||
* BOTH
|
||||
* (a) a getPage/tx.getPage call whose balanced argument span has no second
|
||||
* argument at all, OR a conditional second argument whose false branch
|
||||
* is undefined/null/{} — shorthand (`x ? { sourceId } : undefined`) and
|
||||
* expanded (`x ? { sourceId: x } : undefined`) forms alike (any-source
|
||||
* when unset — the read half of the bug),
|
||||
* AND
|
||||
* (b) any write-path call: putPage( / importFromContent( / importFromFile(.
|
||||
*
|
||||
* The fix pattern (operations.ts): `getPage(slug, { sourceId: x ?? 'default' })`
|
||||
* — mirror the write's schema default on the read.
|
||||
*
|
||||
* Opt-out: a `gbrain-allow-unscoped-getpage: <reason>` comment ANYWHERE in the
|
||||
* getPage call span or on the line above it (for genuinely read-only,
|
||||
* first-match-semantics callers).
|
||||
*
|
||||
* Exit 0 = clean, 1 = violations. Runs under node or bun.
|
||||
*/
|
||||
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
// Default scan roots; overridable via argv so the guard's self-test can point
|
||||
// it at fixtures (`node check-getpage-scoped-write.mjs /tmp/fixtures`).
|
||||
const ROOTS = process.argv.slice(2).length > 0 ? process.argv.slice(2) : ['src'];
|
||||
|
||||
const GETPAGE_RE = /\.\s*getPage\s*(?:<[^>;]*>)?\s*\(/g;
|
||||
const WRITE_RE = /\b(putPage|importFromContent|importFromFile)\s*(?:<[^>;]*>)?\s*\(/;
|
||||
const OPT_OUT = 'gbrain-allow-unscoped-getpage';
|
||||
|
||||
/** Walk from the '(' at openIdx and return [start,end) of the balanced span,
|
||||
* respecting strings, template literals, and comments. */
|
||||
function findSpan(src, openIdx) {
|
||||
let depth = 0;
|
||||
let mode = 'code'; // code | line | block | sq | dq | tpl
|
||||
for (let i = openIdx; i < src.length; i++) {
|
||||
const c = src[i];
|
||||
const n = src[i + 1];
|
||||
if (mode === 'line') { if (c === '\n') mode = 'code'; continue; }
|
||||
if (mode === 'block') { if (c === '*' && n === '/') { mode = 'code'; i++; } continue; }
|
||||
if (mode === 'sq') { if (c === '\\') { i++; continue; } if (c === "'") mode = 'code'; continue; }
|
||||
if (mode === 'dq') { if (c === '\\') { i++; continue; } if (c === '"') mode = 'code'; continue; }
|
||||
if (mode === 'tpl') { if (c === '\\') { i++; continue; } if (c === '`') mode = 'code'; continue; }
|
||||
if (c === '/' && n === '/') { mode = 'line'; i++; continue; }
|
||||
if (c === '/' && n === '*') { mode = 'block'; i++; continue; }
|
||||
if (c === "'") { mode = 'sq'; continue; }
|
||||
if (c === '"') { mode = 'dq'; continue; }
|
||||
if (c === '`') { mode = 'tpl'; continue; }
|
||||
if (c === '(') depth++;
|
||||
else if (c === ')') { depth--; if (depth === 0) return [openIdx + 1, i]; }
|
||||
}
|
||||
return [openIdx + 1, src.length];
|
||||
}
|
||||
|
||||
/** Blank out comments so commented examples don't trip the probes. */
|
||||
function stripComments(s) {
|
||||
return s.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
}
|
||||
|
||||
/** Split a balanced span into top-level arguments (commas at depth 0 only). */
|
||||
function topLevelArgs(span) {
|
||||
const args = [];
|
||||
let depth = 0;
|
||||
let mode = 'code';
|
||||
let cur = '';
|
||||
for (let i = 0; i < span.length; i++) {
|
||||
const c = span[i];
|
||||
const n = span[i + 1];
|
||||
if (mode === 'line') { if (c === '\n') mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'block') { if (c === '*' && n === '/') { mode = 'code'; cur += '*/'; i++; continue; } cur += c; continue; }
|
||||
if (mode === 'sq') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === "'") mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'dq') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === '"') mode = 'code'; cur += c; continue; }
|
||||
if (mode === 'tpl') { if (c === '\\') { cur += c + (n ?? ''); i++; continue; } if (c === '`') mode = 'code'; cur += c; continue; }
|
||||
if (c === '/' && n === '/') { mode = 'line'; cur += c; continue; }
|
||||
if (c === '/' && n === '*') { mode = 'block'; cur += c; continue; }
|
||||
if (c === "'") { mode = 'sq'; cur += c; continue; }
|
||||
if (c === '"') { mode = 'dq'; cur += c; continue; }
|
||||
if (c === '`') { mode = 'tpl'; cur += c; continue; }
|
||||
if (c === '(' || c === '[' || c === '{') depth++;
|
||||
else if (c === ')' || c === ']' || c === '}') depth--;
|
||||
else if (c === ',' && depth === 0) { args.push(cur); cur = ''; continue; }
|
||||
cur += c;
|
||||
}
|
||||
if (cur.trim().length > 0) args.push(cur);
|
||||
return args;
|
||||
}
|
||||
|
||||
/** True when the getPage second argument is the any-source-when-unset shape. */
|
||||
function isUnscopedRead(span) {
|
||||
const args = topLevelArgs(span);
|
||||
if (args.length < 2) return true; // no opts at all → unscoped
|
||||
const opts = stripComments(args[1]).trim();
|
||||
// Ternary opts whose false branch is undefined/null/{} — any-source when
|
||||
// unset. Covers BOTH the shorthand (`x ? { sourceId } : undefined`) and the
|
||||
// expanded form (`x ? { sourceId: x } : undefined`): the object-literal
|
||||
// colon in the expanded form defeated a naive [^:]* regex, so this checks
|
||||
// "mentions sourceId + ends in a bare-empty false branch" instead.
|
||||
if (opts.includes('sourceId') && /\?[\s\S]*:\s*(undefined|null|\{\s*\})\s*$/.test(opts)) return true;
|
||||
if (/^(undefined|null|\{\s*\})$/.test(opts)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
|
||||
function scanFile(file) {
|
||||
const src = readFileSync(file, 'utf8');
|
||||
if (!WRITE_RE.test(stripComments(src))) return; // no write path in this file → read-only semantics allowed
|
||||
GETPAGE_RE.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = GETPAGE_RE.exec(src))) {
|
||||
const openIdx = m.index + m[0].length - 1;
|
||||
const [s, e] = findSpan(src, openIdx);
|
||||
const span = src.slice(s, e);
|
||||
// Opt-out marker inside the span, on the lines just before the call, or
|
||||
// in a trailing comment on the closing-paren line.
|
||||
const before = src.slice(Math.max(0, m.index - 300), m.index);
|
||||
const afterEnd = src.indexOf('\n', e);
|
||||
const tail = src.slice(e, afterEnd === -1 ? src.length : afterEnd);
|
||||
if (
|
||||
span.includes(OPT_OUT) ||
|
||||
before.split('\n').slice(-3).join('\n').includes(OPT_OUT) ||
|
||||
tail.includes(OPT_OUT)
|
||||
) continue;
|
||||
if (!isUnscopedRead(span)) continue;
|
||||
const line = src.slice(0, m.index).split('\n').length;
|
||||
violations.push(
|
||||
`${file}:${line} unscoped getPage(...) in a file that also writes (putPage/importFromContent) — ` +
|
||||
`scope the read to the write's source: getPage(slug, { sourceId: x ?? 'default' })`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function walk(dir) {
|
||||
let ents;
|
||||
try { ents = readdirSync(dir); } catch { return; }
|
||||
for (const ent of ents) {
|
||||
if (ent === 'node_modules') continue;
|
||||
const p = join(dir, ent);
|
||||
const st = statSync(p);
|
||||
if (st.isDirectory()) walk(p);
|
||||
else if (p.endsWith('.ts') && !p.endsWith('.test.ts')) scanFile(p);
|
||||
}
|
||||
}
|
||||
|
||||
for (const root of ROOTS) walk(root);
|
||||
|
||||
if (violations.length) {
|
||||
console.error('Unscoped-getPage-with-write violations (source-isolation bug class):\n');
|
||||
for (const v of violations) console.error(' ' + v);
|
||||
console.error(
|
||||
`\n${violations.length} violation(s). Fix: pass { sourceId: x ?? 'default' } on the read ` +
|
||||
`(mirrors putPage's schema default), or mark genuinely read-only first-match calls with ` +
|
||||
`a '${OPT_OUT}: <reason>' comment.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('check-getpage-scoped-write: clean (no unscoped getPage in write-path files)');
|
||||
@@ -66,3 +66,4 @@ check-module-size.sh scanner yes committed per-file line ceilings (module-size-l
|
||||
check-structural-manifest.sh buildfresh exempt regenerate+diff of structural-suites.tsv (classify-tests.ts); the diff IS the self-test
|
||||
check-opencode-pin.sh repostate exempt pin-stamp drift check (OPENCODE-CLI-PIN.md stamps vs heavy-tests opencode-door env); own bun guard tests in test/check-bootstrap-guards.test.ts
|
||||
check-pin-doc-privacy.sh repostate exempt PIN-doc placeholder discipline (no operator paths/key material/emails in docs/mcp/*-CLI-PIN.md); own bun guard tests in test/check-bootstrap-guards.test.ts
|
||||
check-getpage-scoped-write.mjs scanner yes unscoped-getPage + write co-occurrence scanner (source-isolation bug class); argv root override; fixtures under test/fixtures/guards/; also in verify CHECKS
|
||||
|
||||
|
@@ -3,33 +3,33 @@
|
||||
# Raising a ceiling is a conscious, reviewer-visible act. Lower ceilings in
|
||||
# the same commit as any peel (the guard fails on >50 lines of stale slack).
|
||||
# Columns: path max_lines policy note
|
||||
src/commands/doctor.ts 4205 ratchet peel target: containment sprint C8-C13
|
||||
src/commands/doctor.ts 4270 ratchet peel target: containment sprint C8-C13; grown v0.46.11.0 five-issue wave
|
||||
src/core/operations.ts 303 ratchet peel target: containment sprint C4-C7
|
||||
src/core/postgres-engine.ts 5734 ratchet peel target: containment sprint C15
|
||||
src/core/pglite-engine.ts 5625 ratchet peel target: containment sprint C15
|
||||
src/core/postgres-engine.ts 5770 ratchet peel target: containment sprint C15; grown v0.46.11.0 five-issue wave
|
||||
src/core/pglite-engine.ts 5660 ratchet peel target: containment sprint C15; grown v0.46.11.0 five-issue wave
|
||||
src/core/migrate.ts 668 region-exempt append-only MIGRATIONS array grows freely; runner logic is ratcheted
|
||||
src/commands/sync.ts 4121 ratchet peel target: containment sprint C13-C14
|
||||
src/commands/sync.ts 4300 ratchet peel target: containment sprint C13-C14; grown v0.46.11.0 five-issue wave
|
||||
src/core/ai/gateway.ts 4116 ratchet watchlist
|
||||
src/cli.ts 3337 ratchet watchlist
|
||||
src/core/cycle.ts 2933 ratchet
|
||||
src/commands/serve-http.ts 2836 ratchet
|
||||
src/commands/jobs.ts 2839 ratchet
|
||||
src/commands/jobs.ts 2950 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/core/search/hybrid.ts 2479 ratchet
|
||||
src/core/engine.ts 2343 ratchet
|
||||
src/commands/autopilot.ts 2301 ratchet
|
||||
src/commands/extract.ts 2161 ratchet
|
||||
src/commands/extract-conversation-facts.ts 1968 ratchet
|
||||
src/core/import-file.ts 1904 ratchet
|
||||
src/core/cycle/synthesize.ts 2616 ratchet
|
||||
src/core/import-file.ts 2000 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/core/cycle/synthesize.ts 2685 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/commands/embed.ts 1963 ratchet
|
||||
src/core/types.ts 1829 ratchet
|
||||
src/commands/skillpack.ts 1763 ratchet
|
||||
src/core/minions/queue.ts 1824 ratchet
|
||||
src/core/minions/queue.ts 2130 ratchet grown v0.46.11.0 five-issue wave
|
||||
src/commands/init.ts 1932 ratchet
|
||||
src/commands/integrations.ts 1675 ratchet
|
||||
src/core/minions/handlers/subagent.ts 1643 ratchet
|
||||
src/commands/bootstrap.ts 1923 ratchet grandfathered at merge (grew past the 1500 cap on master)
|
||||
src/core/minions/worker.ts 1508 ratchet grandfathered at merge (grew past the 1500 cap on master, #4170)
|
||||
src/core/minions/worker.ts 1560 ratchet grandfathered at merge (grew past the 1500 cap on master, #4170); grown v0.46.11.0 five-issue wave
|
||||
src/commands/sources.ts 1586 ratchet
|
||||
src/core/bootstrap/harness.ts 1947 ratchet
|
||||
src/commands/hook.ts 1525 ratchet
|
||||
|
||||
|
@@ -94,6 +94,7 @@ CHECKS=(
|
||||
"check:doc-history"
|
||||
"check:fixture-privacy"
|
||||
"check:source-scope-onboard"
|
||||
"check:getpage-scope"
|
||||
"check:no-double-retry"
|
||||
"check:batch-audit-site"
|
||||
"check:engine-dynamic-import"
|
||||
|
||||
@@ -103,6 +103,7 @@ test/features.test.ts CLI routing 2 bun-file
|
||||
test/filing-rules-resolution.serial.test.ts per-source filing-rules resolution 3 readFileSync
|
||||
test/fix-wave-structural.test.ts #2084 — cli.ts owns process-exit teardown via finishCliTeardown 4 readFileSync
|
||||
test/fix-wave-structural.test.ts WAL-repair wave structural pins (#223/#2575) 4 readFileSync
|
||||
test/fix-wave-structural.test.ts five-issue fix wave — integrity progress is (source_id, slug)-keyed 1 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1077 — admin register-client supports PKCE public clients 1 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1090 — admin embed two-tier resolution 3 readFileSync
|
||||
test/fix-wave-structural.test.ts v0.36.1.x #1124 — query --no-expand actually negates expand 1 readFileSync
|
||||
|
||||
|
Can't render this file because it contains an unexpected character in line 27 and column 63.
|
@@ -236,6 +236,18 @@ Queue/priority/retry tuning is not exposed by `gbrain agent run`; submit the
|
||||
raw `subagent` handler via `gbrain jobs submit` (requires CLI trust) if you
|
||||
need those knobs.
|
||||
|
||||
**Admission control (v0.46.11.0).** Identical parentless `subagent` submits
|
||||
(same owner lane, payload, and execution options) coalesce onto the existing
|
||||
waiting job: `gbrain agent run` prints `coalesced` with the matched job id,
|
||||
and the `submit_agent` MCP response carries `coalesced: true`. Treat that as
|
||||
success — monitor the matched id, do NOT resubmit. Jobs still waiting after
|
||||
the TTL (48h default for `subagent`; `minions.ttl_waiting_hours.<name>`)
|
||||
are cancelled with reason prefix `waiting_ttl_expired`. If an operator has
|
||||
configured a waiting quota (`minions.quota_max_waiting.<name>`), a submit
|
||||
past the cap returns a structured, retryable `rate_limited` error — back
|
||||
off and check `gbrain jobs stats` for a `DIVERGENT QUEUE` line before
|
||||
retrying.
|
||||
|
||||
## Phase 2: Monitor
|
||||
|
||||
```
|
||||
@@ -488,6 +500,7 @@ Total tokens so far: 4.3k
|
||||
- Don't spawn a Minion for a single search query (use search tool directly)
|
||||
- Don't fire-and-forget without checking results
|
||||
- Don't spawn > 5 concurrent agents without checking `gbrain jobs stats` first
|
||||
- Don't resubmit when a submit reports `coalesced` — the work is already queued; monitor the matched job id instead
|
||||
- For subagent work, don't use `sessions_spawn` with `runtime: "subagent"` when Minions is available (use `gbrain agent run` instead)
|
||||
- Don't poll `get_job` in a tight loop (use `get_job_progress` for lightweight checks)
|
||||
- Don't run an operation expected to exceed ~2 minutes as a bare background shell — it dies with the session; route through the Durable execution ladder
|
||||
|
||||
@@ -177,8 +177,9 @@ Validate before sync:
|
||||
gbrain schema lint --with-db
|
||||
```
|
||||
|
||||
The `--with-db` flag opts into the 2 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`) that detect
|
||||
The `--with-db` flag opts into the 4 DB-aware rules
|
||||
(`extractable_empty_corpus`, `mutation_count_anomaly`,
|
||||
`stored_type_is_alias`, `stored_type_undeclared`) that detect
|
||||
mis-declared types you'd otherwise discover only at runtime.
|
||||
|
||||
### Phase 5 — Sync (backfill existing pages with the new types)
|
||||
|
||||
@@ -133,7 +133,7 @@
|
||||
"migrations/v0.8.1.md": "fad7341cfb5e02545fb8a23221d12ab395fc3d8db15d1d8ee8a18844aea6563a",
|
||||
"migrations/v0.9.0.md": "773fab0a8d7f330576265a3f510c1f318f47789b6136c46d43e08121acbc20eb",
|
||||
"migrations/v0.9.1.md": "75761bad6c0ad37b69ec8197c6a678bb6a1484f9a76e4b70f2d1e86dc80102b3",
|
||||
"minion-orchestrator/SKILL.md": "5ddeff9bde80ef7fe4990c97220338ffc9ba0d2126eceaed7b4b6a3eb8b0fa18",
|
||||
"minion-orchestrator/SKILL.md": "a0319963481eae87466871b423757d6a82561d241578304de55ed765cb970356",
|
||||
"minion-orchestrator/routing-eval.jsonl": "501ed2e19cb16847ff8425219d246b7a774de1accd42cb28fd44edbb64204992",
|
||||
"perplexity-research/SKILL.md": "c25f5c471cbe3c6e0f975d8397e8382b00a85f8aa75302231d53c52855369e97",
|
||||
"perplexity-research/routing-eval.jsonl": "f1a40d87e710d5d2acd602a372d83f46c95da022b6e635228fffeaacb3bb2b27",
|
||||
@@ -149,7 +149,7 @@
|
||||
"research-compendium/routing-eval.jsonl": "7446cdcaf9c43fe2e20aaf129a705f13a7743f5f14455a7a21c663572def9078",
|
||||
"resolve-before-asking/SKILL.md": "1882c45b2e603bbb1e251d388cc2682270ee7eae99211d5a5322430f4667fb39",
|
||||
"resolve-before-asking/routing-eval.jsonl": "bac1bcf30337f5255ef4ce1a2a8a2b38d58ebcd576503c483190c79ec6e69489",
|
||||
"schema-author/SKILL.md": "4ac1c8fd08800f3728ec55cdc98e97a5aa618a26b753a0fb38c0df9624b66e06",
|
||||
"schema-author/SKILL.md": "1dd11a44dabcb7d57244be4cf5f4903feb9d146bcbb4363fc150daefc01d04ce",
|
||||
"schema-unify/SKILL.md": "e9ac84018d673d35f749a1f74380d635512308fa50951995a7cb339ab4c85fa6",
|
||||
"setup/SKILL.md": "7f11b70ed89d4bff87096aa7e7bb0d41191eb46682066f3b2cffa7a326b56330",
|
||||
"signal-detector/SKILL.md": "c85772f129b3a5b5b0edfa191e11b1048942e52b7472bbaea224e7188f8af75a",
|
||||
|
||||
+31
-5
@@ -16,6 +16,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../core/minions/admission.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../core/minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, SubagentHandlerData, AggregatorHandlerData } from '../core/minions/types.ts';
|
||||
import { resolveSourceId, ALL_SOURCES } from '../core/source-resolver.ts';
|
||||
@@ -313,7 +314,13 @@ export async function runAgentRun(engine: BrainEngine, args: string[]): Promise<
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
|
||||
process.stderr.write(`submitted: job ${job.id} (subagent)\n`);
|
||||
// Honest-dispatch at the interactive surface (codex re-review): a
|
||||
// param-coalesced submit returns an EXISTING waiting job — printing
|
||||
// 'submitted' would tell the operator a new run was queued when it wasn't.
|
||||
process.stderr.write(job.coalesced === true
|
||||
? `coalesced: identical params matched existing waiting job ${job.id} (subagent). ` +
|
||||
`Vary the prompt/params or pass a fresh idempotency key for an independent run.\n`
|
||||
: `submitted: job ${job.id} (subagent)\n`);
|
||||
|
||||
if (flags.detach || !flags.follow) {
|
||||
process.stdout.write(String(job.id) + '\n');
|
||||
@@ -361,7 +368,9 @@ async function runFanout(engine: BrainEngine, queue: MinionQueue, flags: RunFlag
|
||||
const job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
process.stderr.write(`submitted: job ${job.id} (single-entry manifest short-circuit)\n`);
|
||||
process.stderr.write(job.coalesced === true
|
||||
? `coalesced: identical params matched existing waiting job ${job.id} (single-entry manifest short-circuit).\n`
|
||||
: `submitted: job ${job.id} (single-entry manifest short-circuit)\n`);
|
||||
if (flags.detach || !flags.follow) { process.stdout.write(`${job.id}\n`); return; }
|
||||
await followJob(engine, queue, job.id, flags.timeoutMs);
|
||||
return;
|
||||
@@ -394,9 +403,26 @@ async function runFanout(engine: BrainEngine, queue: MinionQueue, flags: RunFlag
|
||||
max_stalled: 3,
|
||||
};
|
||||
if (flags.timeoutMs) submitOpts.timeout_ms = flags.timeoutMs;
|
||||
const child = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
let child;
|
||||
try {
|
||||
child = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
} catch (e) {
|
||||
// Admission quota mid-fanout: a partial tree (some children submitted,
|
||||
// children_ids never written) would leave the aggregator torn — cancel
|
||||
// the WHOLE tree (cascades to already-submitted children) and surface
|
||||
// the quota message. All-or-nothing beats a wedged aggregator.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
await queue.cancelJob(aggregator.id).catch(() => {});
|
||||
console.error(
|
||||
`fanout aborted at child ${childIds.length + 1}/${manifest.length}: ${e.message}\n` +
|
||||
`Aggregator ${aggregator.id} and its ${childIds.length} submitted child(ren) were cancelled.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
childIds.push(child.id);
|
||||
}
|
||||
|
||||
|
||||
+155
-37
@@ -10,13 +10,16 @@
|
||||
* gbrain check-backlinks fix --dry-run # preview fixes
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync, statSync, lstatSync, existsSync } from 'fs';
|
||||
import { readFileSync, readdirSync, statSync, lstatSync, existsSync } from 'fs';
|
||||
import { join, relative, basename } from 'path';
|
||||
import { extractEntityRefs as canonicalExtractEntityRefs } from '../core/link-extraction.ts';
|
||||
import { createProgress, startHeartbeat } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { parseMarkdown, frontmatterBodyOffset } from '../core/markdown.ts';
|
||||
import { atomicWriteFileSync } from '../core/atomic-write.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
|
||||
interface BacklinkGap {
|
||||
export interface BacklinkGap {
|
||||
/** The page that mentions the entity */
|
||||
sourcePage: string;
|
||||
/** The entity page that's missing the back-link */
|
||||
@@ -132,10 +135,77 @@ export function findBacklinkGaps(brainDir: string): BacklinkGap[] {
|
||||
return gaps;
|
||||
}
|
||||
|
||||
/** Fix back-link gaps by appending timeline entries to target pages */
|
||||
export function fixBacklinkGaps(brainDir: string, gaps: BacklinkGap[], dryRun: boolean = false): number {
|
||||
/** Per-run outcome of the fixer: entries inserted + per-file skip reasons. */
|
||||
export interface BacklinkFixOutcome {
|
||||
fixed: number;
|
||||
skipped: Array<{ page: string; reason: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation codes that make a file UNSAFE to edit: the fence/YAML itself is
|
||||
* broken (or the offset math would be unreliable), so any body insertion could
|
||||
* worsen the damage. Deliberately NOT in this set: MISSING_OPEN (a legacy page
|
||||
* with no frontmatter at all has no fence to corrupt — the whole file is body
|
||||
* and stays fixable) and the content-quality lint codes (NESTED_QUOTES,
|
||||
* NON_STRING_FIELD, EMPTY_FRONTMATTER, SLUG_MISMATCH) whose presence doesn't
|
||||
* affect where the body starts.
|
||||
*/
|
||||
const EDIT_BLOCKING_CODES = new Set(['YAML_PARSE', 'MISSING_CLOSE', 'NULL_BYTES']);
|
||||
|
||||
function firstEditBlockingError(content: string, filePath: string): string | null {
|
||||
const parsed = parseMarkdown(content, filePath, { validate: true });
|
||||
const blocking = (parsed.errors ?? []).find(e => EDIT_BLOCKING_CODES.has(e.code));
|
||||
return blocking ? `${blocking.code}: ${blocking.message}` : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a timeline entry into the body of `content`, never touching bytes
|
||||
* before `bodyStart`. The `## Timeline` heading is matched only as a real
|
||||
* heading line at/after bodyStart (CRLF-tolerant), so a `## Timeline` string
|
||||
* inside YAML frontmatter, a `### Timeline` sub-heading, or a
|
||||
* `## Timeline (2026)` variant never anchors the insertion. With multiple real
|
||||
* headings, the FIRST one wins deterministically (post-validation guards the
|
||||
* result either way). Exported for direct unit tests.
|
||||
*/
|
||||
export function insertTimelineEntry(content: string, bodyStart: number, entry: string): string {
|
||||
const bodySlice = content.slice(bodyStart);
|
||||
const headingMatch = /^## Timeline[ \t]*\r?$/m.exec(bodySlice);
|
||||
|
||||
if (!headingMatch) {
|
||||
// No real Timeline heading in the body — append a fresh section.
|
||||
return content.trimEnd() + '\n\n## Timeline\n\n' + entry + '\n';
|
||||
}
|
||||
|
||||
const headingAbs = bodyStart + headingMatch.index;
|
||||
const headingLineEnd = content.indexOf('\n', headingAbs);
|
||||
const sectionStart = headingLineEnd === -1 ? content.length : headingLineEnd + 1;
|
||||
|
||||
const nextHeading = /^## /m.exec(content.slice(sectionStart));
|
||||
if (nextHeading) {
|
||||
const insertAt = sectionStart + nextHeading.index;
|
||||
return content.slice(0, insertAt) + entry + '\n' + content.slice(insertAt);
|
||||
}
|
||||
return content.trimEnd() + '\n' + entry + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix back-link gaps by inserting timeline entries into target pages.
|
||||
*
|
||||
* Safety pipeline per target file (each failure isolates to that file and is
|
||||
* reported in `skipped` — one bad page can't kill the batch or corrupt itself):
|
||||
* lock (withPageLock) → read → pre-validate (skip if the fence/YAML is
|
||||
* already broken) → insert after the frontmatter-safe body offset →
|
||||
* post-validate the candidate → atomic write (tmp+fsync+rename) that
|
||||
* re-validates the on-disk bytes before the rename.
|
||||
*/
|
||||
export async function fixBacklinkGaps(
|
||||
brainDir: string,
|
||||
gaps: BacklinkGap[],
|
||||
dryRun: boolean = false,
|
||||
opts?: { lockRoot?: string },
|
||||
): Promise<BacklinkFixOutcome> {
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
let fixed = 0;
|
||||
const outcome: BacklinkFixOutcome = { fixed: 0, skipped: [] };
|
||||
|
||||
// Group gaps by target page to batch writes
|
||||
const byTarget = new Map<string, BacklinkGap[]>();
|
||||
@@ -149,42 +219,62 @@ export function fixBacklinkGaps(brainDir: string, gaps: BacklinkGap[], dryRun: b
|
||||
const targetPath = join(brainDir, targetPage);
|
||||
if (!existsSync(targetPath)) continue;
|
||||
|
||||
let content = readFileSync(targetPath, 'utf-8');
|
||||
const lockKey = targetPage.replace(/\.md$/, '');
|
||||
try {
|
||||
await withPageLock(lockKey, async () => {
|
||||
let content = readFileSync(targetPath, 'utf-8');
|
||||
|
||||
for (const gap of targetGaps) {
|
||||
// Compute relative path from target to source
|
||||
const targetDir = targetPage.split('/').slice(0, -1);
|
||||
const sourceDir = gap.sourcePage.split('/');
|
||||
const depth = targetDir.length;
|
||||
const relPrefix = '../'.repeat(depth);
|
||||
const relPath = relPrefix + gap.sourcePage;
|
||||
|
||||
const entry = buildBacklinkEntry(gap.sourceTitle, relPath, today);
|
||||
|
||||
// Insert into Timeline section
|
||||
if (content.includes('## Timeline')) {
|
||||
const parts = content.split('## Timeline');
|
||||
const afterTimeline = parts[1];
|
||||
const nextSection = afterTimeline.match(/\n## /);
|
||||
if (nextSection) {
|
||||
const insertIdx = parts[0].length + '## Timeline'.length + nextSection.index!;
|
||||
content = content.slice(0, insertIdx) + '\n' + entry + content.slice(insertIdx);
|
||||
} else {
|
||||
content = content.trimEnd() + '\n' + entry + '\n';
|
||||
const preError = firstEditBlockingError(content, targetPath);
|
||||
if (preError) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: `pre-existing invalid frontmatter (${preError}) — file left untouched`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Add Timeline section
|
||||
content = content.trimEnd() + '\n\n## Timeline\n\n' + entry + '\n';
|
||||
}
|
||||
fixed++;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
writeFileSync(targetPath, content);
|
||||
const bodyStart = frontmatterBodyOffset(content);
|
||||
let inserted = 0;
|
||||
for (const gap of targetGaps) {
|
||||
// Compute relative path from target to source
|
||||
const targetDir = targetPage.split('/').slice(0, -1);
|
||||
const depth = targetDir.length;
|
||||
const relPrefix = '../'.repeat(depth);
|
||||
const relPath = relPrefix + gap.sourcePage;
|
||||
|
||||
const entry = buildBacklinkEntry(gap.sourceTitle, relPath, today);
|
||||
content = insertTimelineEntry(content, bodyStart, entry);
|
||||
inserted++;
|
||||
}
|
||||
|
||||
const postError = firstEditBlockingError(content, targetPath);
|
||||
if (postError) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: `edit would invalidate page (${postError}) — aborted, file left untouched`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
atomicWriteFileSync(targetPath, content, {
|
||||
verify: (onDisk) => {
|
||||
const diskError = firstEditBlockingError(onDisk, targetPath);
|
||||
if (diskError) throw new Error(`on-disk validation failed (${diskError})`);
|
||||
},
|
||||
});
|
||||
}
|
||||
outcome.fixed += inserted;
|
||||
}, { timeoutMs: 10_000, lockRoot: opts?.lockRoot });
|
||||
} catch (e) {
|
||||
outcome.skipped.push({
|
||||
page: targetPage,
|
||||
reason: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return fixed;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
export interface BacklinksOpts {
|
||||
@@ -199,6 +289,9 @@ export interface BacklinksResult {
|
||||
fixed: number;
|
||||
pages_affected: number;
|
||||
dryRun: boolean;
|
||||
/** Pages the fixer refused to touch (invalid frontmatter, lock/write errors). */
|
||||
skipped_invalid?: number;
|
||||
skipped_pages?: Array<{ page: string; reason: string }>;
|
||||
}
|
||||
|
||||
export interface ParsedBacklinksArgs {
|
||||
@@ -263,8 +356,27 @@ export async function runBacklinksCore(opts: BacklinksOpts): Promise<BacklinksRe
|
||||
const pagesAffected = new Set(gaps.map(g => g.targetPage)).size;
|
||||
|
||||
if (opts.action === 'fix' && gaps.length > 0) {
|
||||
const fixed = fixBacklinkGaps(opts.dir, gaps, !!opts.dryRun);
|
||||
return { action: 'fix', gaps_found: gaps.length, fixed, pages_affected: pagesAffected, dryRun: !!opts.dryRun };
|
||||
// Locks + per-file validation make the fix loop slower than the naive
|
||||
// writer it replaced — run it under its own phase with a heartbeat so
|
||||
// agents see forward progress (the scan phase above already finished).
|
||||
progress.start('backlinks.fix');
|
||||
const fixHb = startHeartbeat(progress, 'applying back-link fixes…');
|
||||
let fixOutcome: BacklinkFixOutcome;
|
||||
try {
|
||||
fixOutcome = await fixBacklinkGaps(opts.dir, gaps, !!opts.dryRun);
|
||||
} finally {
|
||||
fixHb();
|
||||
progress.finish();
|
||||
}
|
||||
return {
|
||||
action: 'fix',
|
||||
gaps_found: gaps.length,
|
||||
fixed: fixOutcome.fixed,
|
||||
pages_affected: pagesAffected,
|
||||
dryRun: !!opts.dryRun,
|
||||
skipped_invalid: fixOutcome.skipped.length,
|
||||
skipped_pages: fixOutcome.skipped,
|
||||
};
|
||||
}
|
||||
return { action: opts.action, gaps_found: gaps.length, fixed: 0, pages_affected: pagesAffected, dryRun: !!opts.dryRun };
|
||||
}
|
||||
@@ -310,6 +422,12 @@ export async function runBacklinks(args: string[]) {
|
||||
} else {
|
||||
const label = result.dryRun ? '(dry run) ' : '';
|
||||
console.log(`${label}Fixed ${result.fixed} missing back-link(s) across ${result.pages_affected} page(s).`);
|
||||
if (result.skipped_pages && result.skipped_pages.length > 0) {
|
||||
console.log(`\nSkipped ${result.skipped_pages.length} page(s):`);
|
||||
for (const s of result.skipped_pages) {
|
||||
console.log(` ${s.page}: ${s.reason}`);
|
||||
}
|
||||
}
|
||||
if (result.dryRun) {
|
||||
console.log('\nRe-run without --dry-run to apply.');
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ async function runScan(
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
const page = await engine.getPage(slug);
|
||||
const page = await engine.getPage(slug); // gbrain-allow-unscoped-getpage: read-only scan CLI with no source parameter; first-match semantics documented
|
||||
if (!page) {
|
||||
process.stderr.write(
|
||||
`[conversation-parser scan] page not found: ${slug}\n`,
|
||||
|
||||
@@ -1128,6 +1128,44 @@ export async function buildChecks(
|
||||
// Best-effort; audit-log read failure shouldn't stop doctor.
|
||||
}
|
||||
|
||||
// 3d.05 Malformed-path pages. DB pages whose backing FILENAME contains
|
||||
// bracket/control characters (markdown-link syntax as a literal filename).
|
||||
// Sync refuses to import such markdown paths; this check is the discovery
|
||||
// surface for rows ingested before that gate. Two-tier remediation matches
|
||||
// core/sync.ts: POISONED rows (`](`/control chars) reconcile away on a full
|
||||
// sync; bare-bracket rows are kept (deleting them while their file exists
|
||||
// would be data loss) and need a rename + re-sync.
|
||||
if (engine) {
|
||||
try {
|
||||
const { hasMalformedPathSegment, isPoisonedPath } = await import('../core/sync.ts');
|
||||
const candidates = await engine.executeRaw<{ slug: string; source_id: string; source_path: string }>(
|
||||
`SELECT slug, source_id, source_path FROM pages
|
||||
WHERE source_path IS NOT NULL AND deleted_at IS NULL
|
||||
AND (source_path LIKE '%[%' OR source_path LIKE '%]%'
|
||||
OR source_path ~ '[[:cntrl:]]')`,
|
||||
[],
|
||||
);
|
||||
const malformed = candidates.filter(r => hasMalformedPathSegment(r.source_path));
|
||||
if (malformed.length > 0) {
|
||||
const poisoned = malformed.filter(r => isPoisonedPath(r.source_path)).length;
|
||||
const bare = malformed.length - poisoned;
|
||||
const preview = malformed.slice(0, 3).map(r => r.slug).join(', ');
|
||||
checks.push({
|
||||
name: 'malformed_path_pages',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${malformed.length} page(s) backed by malformed filenames (bracket/control ` +
|
||||
`characters) pollute search: ${preview}` +
|
||||
`${malformed.length > 3 ? `, and ${malformed.length - 3} more` : ''}. ` +
|
||||
(poisoned > 0 ? `${poisoned} junk row(s): run a full 'gbrain sync' to reconcile them away. ` : '') +
|
||||
(bare > 0 ? `${bare} bare-bracket row(s) are kept — rename the backing file(s) and re-sync.` : ''),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort; a schema without source_path shouldn't stop doctor.
|
||||
}
|
||||
}
|
||||
|
||||
// 3d.1 Nightly quality probe (v0.40.1.0 Track D / T7). Reads the last
|
||||
// 7 days of quality-probe-YYYY-Www.jsonl audit events. SKIPPED with
|
||||
// paste-ready enable hint when the feature is opt-in disabled (default).
|
||||
|
||||
@@ -202,6 +202,61 @@ export async function computeQueueHealthCheck(
|
||||
`→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).`
|
||||
);
|
||||
}
|
||||
// Queue divergence: per-type intake structurally exceeds useful drain
|
||||
// (completions keyed on finished_at) while a real backlog waits. Same
|
||||
// env thresholds as the `jobs stats` DIVERGENT scream so the two
|
||||
// advisory surfaces agree. Cancellations (incl. the waiting-TTL sweep)
|
||||
// are deliberately NOT counted as drain — outflow is not work.
|
||||
try {
|
||||
const { TTL_REASON_PREFIX, safeConfigSegment } = await import('../../../core/minions/admission.ts');
|
||||
const { sanitizeTypeForDisplay } = await import('../../../core/schema-pack/type-usage.ts');
|
||||
const divergenceRatio = resolveEnvNumber('GBRAIN_QUEUE_DIVERGENCE_RATIO', 2);
|
||||
const divergenceMinWaiting = resolveEnvNumber('GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING', 50);
|
||||
const divRows = await engine.executeRaw<{ name: string; intake: string; completed: string; waiting: string }>(
|
||||
`SELECT w.name,
|
||||
COALESCE(i.intake, '0') AS intake,
|
||||
COALESCE(c.completed, '0') AS completed,
|
||||
w.waiting
|
||||
FROM (SELECT name, count(*)::text AS waiting FROM minion_jobs
|
||||
WHERE status = 'waiting' GROUP BY name) w
|
||||
LEFT JOIN (SELECT name, count(*)::text AS intake FROM minion_jobs
|
||||
WHERE created_at > now() - interval '24 hours' GROUP BY name) i ON i.name = w.name
|
||||
LEFT JOIN (SELECT name, count(*)::text AS completed FROM minion_jobs
|
||||
WHERE finished_at > now() - interval '24 hours' AND status = 'completed'
|
||||
GROUP BY name) c ON c.name = w.name`,
|
||||
);
|
||||
for (const r of divRows) {
|
||||
const waiting = parseInt(r.waiting, 10);
|
||||
const intake = parseInt(r.intake, 10);
|
||||
const completed = parseInt(r.completed, 10);
|
||||
if (waiting > divergenceMinWaiting && intake > divergenceRatio * Math.max(completed, 1)) {
|
||||
// Job names originate from the MCP-exposed submit surface —
|
||||
// sanitize for display; strict-gate names embedded in the
|
||||
// copy-pasteable config hint.
|
||||
problems.push(
|
||||
`DIVERGENT queue type '${sanitizeTypeForDisplay(r.name)}': intake ${intake}/24h vs ${completed} completed/24h, ` +
|
||||
`${waiting} waiting — the backlog grows structurally. Reduce intake, raise drain, or cap ` +
|
||||
`admission: \`gbrain config set minions.quota_max_waiting.${safeConfigSegment(r.name) ?? '<job-name>'} <n>\`. See \`gbrain jobs stats\`.`
|
||||
);
|
||||
}
|
||||
}
|
||||
// Waiting-TTL cancellations mean the divergence is being SHREDDED, not
|
||||
// worked — that's operating as designed but the operator must know.
|
||||
const ttlRows = await engine.executeRaw<{ name: string; count: string }>(
|
||||
`SELECT name, count(*)::text AS count FROM minion_jobs
|
||||
WHERE status = 'cancelled' AND error_text LIKE $1
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
GROUP BY name`,
|
||||
[`${TTL_REASON_PREFIX}%`],
|
||||
);
|
||||
for (const r of ttlRows) {
|
||||
problems.push(
|
||||
`waiting-TTL cancelled ${r.count} '${sanitizeTypeForDisplay(r.name)}' job(s) in the last 24h (queued work expired ` +
|
||||
`unclaimed — intake still exceeds drain). Tune: \`gbrain config set ` +
|
||||
`minions.ttl_waiting_hours.${safeConfigSegment(r.name) ?? '<job-name>'} <hours|0>\`.`
|
||||
);
|
||||
}
|
||||
} catch { /* best-effort — divergence probes never break doctor */ }
|
||||
if (promptTooLongCount > 0) {
|
||||
problems.push(
|
||||
`${promptTooLongCount} subagent job(s) dead-lettered with prompt_too_long in last 24h. ` +
|
||||
|
||||
+96
-5
@@ -8,6 +8,8 @@ import { loadConfig, gbrainPath } from '../core/config.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import {
|
||||
hasMalformedPathSegment,
|
||||
sanitizePathForDisplay,
|
||||
isCodeFilePath,
|
||||
isMarkdownFilePath,
|
||||
isImageFilePath as isImageFilePathFromSync,
|
||||
@@ -92,6 +94,10 @@ export interface RunImportResult {
|
||||
errors: number;
|
||||
chunksCreated: number;
|
||||
failures: Array<{ path: string; error: string }>;
|
||||
/** Files dropped by the malformed-filename gate (walker + per-file defense). */
|
||||
malformedSkipped?: number;
|
||||
/** Aggregated alias/undeclared explicit-type warnings (schema.type_warnings). */
|
||||
type_warnings?: Array<{ kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string; count: number }>;
|
||||
}
|
||||
|
||||
export async function runImport(
|
||||
@@ -175,7 +181,7 @@ export async function runImport(
|
||||
}
|
||||
// v0.39 T1.5: load active pack ONCE at runImport entry; thread to every
|
||||
// per-file importFile call below. Codex perf finding #7 — never per-file.
|
||||
let importActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
|
||||
let importActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> } | undefined;
|
||||
try {
|
||||
const { loadActivePack } = await import('../core/schema-pack/load-active.ts');
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
@@ -275,10 +281,22 @@ export async function runImport(
|
||||
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
|
||||
const _walkT0 = Date.now();
|
||||
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
|
||||
let allFiles = collectSyncableFiles(dir, { strategy, includeGitignored });
|
||||
const malformedExcluded: string[] = [];
|
||||
let allFiles = collectSyncableFiles(dir, {
|
||||
strategy, includeGitignored,
|
||||
onExcluded: (rel) => { malformedExcluded.push(rel); },
|
||||
});
|
||||
console.error(
|
||||
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
|
||||
);
|
||||
if (malformedExcluded.length > 0) {
|
||||
console.error(
|
||||
`[gbrain import] ${malformedExcluded.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
malformedExcluded.slice(0, 20).map(sanitizePathForDisplay).join(', ') +
|
||||
(malformedExcluded.length > 20 ? `, … (+${malformedExcluded.length - 20} more)` : ''),
|
||||
);
|
||||
}
|
||||
const fileTypeLabel = strategy === 'code' ? 'code'
|
||||
: strategy === 'auto' ? 'syncable' : 'markdown';
|
||||
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
|
||||
@@ -327,6 +345,9 @@ export async function runImport(
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
// Per-file malformed skips (defense-in-depth hits inside importFromFile);
|
||||
// the walker-level exclusions are counted separately via malformedExcluded.
|
||||
let malformedFileSkips = 0;
|
||||
let processed = 0;
|
||||
// Time-based checkpoint floor (see the save site below). Chunking cost scales
|
||||
// with paragraph count, not bytes, so a single reference-style file can take
|
||||
@@ -339,6 +360,16 @@ export async function runImport(
|
||||
const errorCounts: Record<string, number> = {};
|
||||
const errorSamples: Record<string, string> = {};
|
||||
const failures: Array<{ path: string; error: string }> = []; // Bug 9
|
||||
// Alias-footgun visibility: aggregate per-file type_warning results once
|
||||
// per distinct type per run (same surface `gbrain sync` carries).
|
||||
const typeWarningCounts = new Map<string, import('../core/schema-pack/type-usage.ts').TypeWarningCount>();
|
||||
const noteTypeWarning = (w: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string } | undefined): void => {
|
||||
if (!w) return;
|
||||
const key = `${w.kind}\t${w.type}`;
|
||||
const cur = typeWarningCounts.get(key);
|
||||
if (cur) cur.count++;
|
||||
else typeWarningCounts.set(key, { ...w, count: 1 });
|
||||
};
|
||||
// #3839: paths that succeeded (imported OR unchanged) this run, keyed the
|
||||
// same way as `failures` above (importRelPath) so a path that failed on a
|
||||
// prior run and now succeeds clears its ledger row instead of staying
|
||||
@@ -373,6 +404,7 @@ export async function runImport(
|
||||
const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true'
|
||||
? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
noteTypeWarning((result as { type_warning?: Parameters<typeof noteTypeWarning>[0] }).type_warning);
|
||||
const _fileMs = Date.now() - _fileT0;
|
||||
if (_fileMs > 5000) {
|
||||
console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`);
|
||||
@@ -386,7 +418,13 @@ export async function runImport(
|
||||
succeededPaths.push(importRelPath); // #3839
|
||||
} else {
|
||||
skipped++;
|
||||
if (result.error && result.error !== 'unchanged') {
|
||||
if (result.skip_reason === 'malformed_path') {
|
||||
// Informational skip (bracket/control-char filename): never a
|
||||
// failure-ledger row, and stable across runs — checkpoint as done.
|
||||
console.error(` Skipped (malformed filename — rename to import): ${sanitizePathForDisplay(relativePath)}`);
|
||||
malformedFileSkips++;
|
||||
completed.add(relativePath);
|
||||
} else if (result.error && result.error !== 'unchanged') {
|
||||
console.error(` Skipped ${relativePath}: ${result.error}`);
|
||||
// Bug 9 — non-"unchanged" skips carry a real error reason.
|
||||
// #774: ledger paths use the slug base so an incremental sync's
|
||||
@@ -591,6 +629,22 @@ export async function runImport(
|
||||
}
|
||||
}
|
||||
|
||||
// Alias/undeclared explicit-type warnings (schema.type_warnings, default on).
|
||||
let typeWarningsEnabled = true;
|
||||
if (typeWarningCounts.size > 0) {
|
||||
try {
|
||||
const v = await engine.getConfig('schema.type_warnings');
|
||||
typeWarningsEnabled = !(v === 'false' || v === '0' || v === 'off');
|
||||
} catch { /* config unavailable → default on */ }
|
||||
if (typeWarningsEnabled) {
|
||||
const { renderTypeWarningSummary } = await import('../core/schema-pack/type-usage.ts');
|
||||
for (const line of renderTypeWarningSummary([...typeWarningCounts.values()])) {
|
||||
console.error(` ${line}`);
|
||||
}
|
||||
console.error(` (silence with: gbrain config set schema.type_warnings false)`);
|
||||
}
|
||||
}
|
||||
|
||||
// Log the ingest
|
||||
await engine.logIngest({
|
||||
source_type: 'directory',
|
||||
@@ -670,7 +724,14 @@ export async function runImport(
|
||||
// this import's to move (its sync anchors live on the `sources` row).
|
||||
}
|
||||
|
||||
return { imported, skipped, errors, chunksCreated, failures };
|
||||
const totalMalformed = malformedExcluded.length + malformedFileSkips;
|
||||
return {
|
||||
imported, skipped, errors, chunksCreated, failures,
|
||||
...(totalMalformed > 0 ? { malformedSkipped: totalMalformed } : {}),
|
||||
...(typeWarningCounts.size > 0 && typeWarningsEnabled
|
||||
? { type_warnings: [...typeWarningCounts.values()] }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -692,6 +753,13 @@ function resolveMaxWalkDepth(): number {
|
||||
interface CollectOpts {
|
||||
strategy?: SyncStrategy;
|
||||
includeGitignored?: boolean;
|
||||
/**
|
||||
* Invoked (with the repo-relative path) for each file dropped by the
|
||||
* malformed-filename gate, on BOTH collection routes. Without this,
|
||||
* directory imports and full syncs silently succeed while omitting the
|
||||
* file — no rename guidance, no skipped count (structured-review finding).
|
||||
*/
|
||||
onExcluded?: (relPath: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -727,6 +795,11 @@ function isCollectibleForWalker(
|
||||
const segments = path.split('/');
|
||||
if (segments.some((seg) => !pruneDir(seg))) return false;
|
||||
|
||||
// Malformed filenames (brackets / control chars — markdown-link syntax as a
|
||||
// literal filename) are rejected on BOTH collection routes, same as
|
||||
// incremental sync's classifySync. Full and incremental must agree.
|
||||
if (hasMalformedPathSegment(path)) return false;
|
||||
|
||||
// Metafiles are directory scaffolding (READMEs / index / log / schema /
|
||||
// resolver), not typed brain pages — same exclusion `sync`'s `isSyncable`
|
||||
// applies. Guards both the FS-walk and the git-fast-path collection routes.
|
||||
@@ -764,6 +837,7 @@ function gitListSyncableFiles(
|
||||
dir: string,
|
||||
strategy: SyncStrategy,
|
||||
multimodalOn: boolean,
|
||||
onExcluded?: (relPath: string) => void,
|
||||
): string[] | null {
|
||||
let stdout: string;
|
||||
try {
|
||||
@@ -778,6 +852,10 @@ function gitListSyncableFiles(
|
||||
const files: string[] = [];
|
||||
for (const rel of stdout.split('\0')) {
|
||||
if (!rel) continue;
|
||||
// Malformed check FIRST (separately from the collectible gate) so the
|
||||
// exclusion is reportable — other filters (strategy, prune, metafile)
|
||||
// are silent by design; this one hides renameable content.
|
||||
if (hasMalformedPathSegment(rel)) { onExcluded?.(rel); continue; }
|
||||
if (!isCollectibleForWalker(rel, strategy, multimodalOn)) continue;
|
||||
const full = join(dir, rel);
|
||||
let st;
|
||||
@@ -823,7 +901,7 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
// PLUS untracked-not-ignored, so uncommitted source is still indexed. Non-git
|
||||
// dirs (or git unavailable) fall through to the FS walk below.
|
||||
if (!opts.includeGitignored) {
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn, opts.onExcluded);
|
||||
if (gitFiles) return gitFiles;
|
||||
}
|
||||
|
||||
@@ -848,6 +926,14 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
// from it. Skips hidden dirs (`.git`, `.raw`, etc.), `node_modules`,
|
||||
// `vendor`, `dist`, `build`, `venv` (#2020), `ops`, and git submodules.
|
||||
if (!pruneDir(entry, d)) continue;
|
||||
// Control-char SEGMENT check at descent time (never legitimate). The
|
||||
// bracket check moved to the per-file RELATIVE-path test below: a
|
||||
// bracket-named DIRECTORY must still be descended for code strategies
|
||||
// (`app/[id]/page.tsx` is ubiquitous framework layout), while markdown
|
||||
// files under it are excluded per-file — mirroring classifySync so full
|
||||
// and incremental sync agree (cross-model adversarial finding).
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (/[\x00-\x1f]/.test(entry)) continue;
|
||||
|
||||
const full = join(d, entry);
|
||||
let stat;
|
||||
@@ -872,6 +958,11 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
visitedInodes.set(inodeKey, true);
|
||||
walk(full, depth + 1);
|
||||
} else if (stat.isFile()) {
|
||||
// Malformed check on the RELATIVE path (this route's
|
||||
// isCollectibleForWalker only sees the basename, which can't catch a
|
||||
// bracket directory segment above a clean-named markdown file).
|
||||
const rel = relative(dir, full);
|
||||
if (hasMalformedPathSegment(rel)) { opts.onExcluded?.(rel); continue; }
|
||||
if (!isCollectibleForWalker(entry, strategy, multimodalOn)) continue;
|
||||
files.push(full);
|
||||
}
|
||||
|
||||
@@ -161,23 +161,46 @@ export function findExternalLinks(compiledTruth: string, slug: string): External
|
||||
|
||||
interface ProgressEntry {
|
||||
slug: string;
|
||||
/**
|
||||
* Source the row belongs to. Progress used to be keyed by slug alone, so a
|
||||
* resume SKIPPED same-slug pages in every other source (the scan iterates
|
||||
* (slug, source_id) pairs). Legacy entries without source_id are treated as
|
||||
* default-source only.
|
||||
*/
|
||||
source_id?: string;
|
||||
status: 'repaired' | 'reviewed' | 'skipped' | 'error';
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** Composite progress key — (source, slug), tab-separated (tabs can't appear in either). */
|
||||
function progressKey(sourceId: string | undefined, slug: string): string {
|
||||
return `${sourceId ?? 'default'}\t${slug}`;
|
||||
}
|
||||
|
||||
function loadProgress(): Set<string> {
|
||||
if (!existsSync(getProgressFile())) return new Set();
|
||||
const seen = new Set<string>();
|
||||
const content = readFileSync(getProgressFile(), 'utf-8');
|
||||
let legacy = 0;
|
||||
for (const line of content.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const entry = JSON.parse(line) as ProgressEntry;
|
||||
seen.add(entry.slug);
|
||||
if (entry.source_id == null) legacy++;
|
||||
seen.add(progressKey(entry.source_id, entry.slug));
|
||||
} catch {
|
||||
/* skip malformed lines */
|
||||
}
|
||||
}
|
||||
if (legacy > 0) {
|
||||
// Pre-(source_id, slug) ledger entries key as default-source only, so a
|
||||
// resume re-scans non-default-source pages they may have covered. Say so
|
||||
// once — a silent partial re-scan reads as "resume is broken".
|
||||
console.error(
|
||||
`integrity: ${legacy} resume-ledger entr${legacy === 1 ? 'y' : 'ies'} predate source tracking; ` +
|
||||
`matching them to the default source only (non-default-source pages re-scan — idempotent, just slower).`,
|
||||
);
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
|
||||
@@ -429,7 +452,18 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
const engine = await connect();
|
||||
const registry = getDefaultRegistry();
|
||||
registerBuiltinResolvers(registry);
|
||||
const writer = new BrainWriter(engine, { strictMode: 'off' });
|
||||
// One writer PER SOURCE: BrainWriter scopes every read/write (and
|
||||
// addTimelineEntry) to its sourceId — a single default-scoped writer used
|
||||
// for every source's pages was the unscoped-check/scoped-write bug class.
|
||||
const writersBySource = new Map<string, BrainWriter>();
|
||||
const writerFor = (sourceId: string): BrainWriter => {
|
||||
let w = writersBySource.get(sourceId);
|
||||
if (!w) {
|
||||
w = new BrainWriter(engine, { strictMode: 'off', sourceId });
|
||||
writersBySource.set(sourceId, w);
|
||||
}
|
||||
return w;
|
||||
};
|
||||
|
||||
const ctx: ResolverContext = {
|
||||
engine,
|
||||
@@ -463,11 +497,12 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
const allRefs = (await engine.listAllPageRefs()).sort((a, b) =>
|
||||
a.slug.localeCompare(b.slug) || a.source_id.localeCompare(b.source_id)
|
||||
);
|
||||
const toScan = allRefs.filter(r => !seen.has(r.slug));
|
||||
const toScan = allRefs.filter(r => !seen.has(progressKey(r.source_id, r.slug)));
|
||||
progress.start('integrity.auto', toScan.length);
|
||||
for (const { slug, source_id } of allRefs) {
|
||||
if (pagesProcessed >= limit) break;
|
||||
if (seen.has(slug)) continue;
|
||||
if (seen.has(progressKey(source_id, slug))) continue;
|
||||
const writer = writerFor(source_id);
|
||||
|
||||
const page = await engine.getPage(slug, { sourceId: source_id });
|
||||
if (!page) continue;
|
||||
@@ -498,26 +533,26 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
// Dry-run must NOT persist 'repaired' — the follow-on real
|
||||
// run needs to revisit these slugs and actually write.
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'repaired', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'repaired', timestamp: new Date().toISOString() });
|
||||
}
|
||||
} else if (result.confidence >= reviewLower) {
|
||||
appendReview({ slug, hit, result, handle });
|
||||
bucketReview++;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'reviewed', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'reviewed', timestamp: new Date().toISOString() });
|
||||
}
|
||||
} else {
|
||||
logSkip({ slug, hit, reason: `confidence ${result.confidence.toFixed(2)} below threshold ${reviewLower}` });
|
||||
bucketSkip++;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
bucketErr++;
|
||||
logSkip({ slug, hit, reason: `resolver error: ${e instanceof Error ? e.message : String(e)}` });
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'error', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'error', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -528,7 +563,7 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
}
|
||||
bucketSkip += hits.length;
|
||||
if (!dryRun) {
|
||||
appendProgress({ slug, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+85
-2
@@ -919,18 +919,101 @@ export async function runJobs(engineOrNull: BrainEngine | null, args: string[]):
|
||||
const statsQueue = parseFlag(args, '--queue') ?? 'default';
|
||||
const stats = await queue.getStats({ queue: statsQueue });
|
||||
|
||||
// Divergence detection: intake (created in window) vs USEFUL drain
|
||||
// (drained_completed — cancellations are outflow, not work; a naive
|
||||
// combined drain self-inflates while the TTL sweep shreds backlog).
|
||||
// Same env-threshold pattern as the wedge line below.
|
||||
const divergenceRatio = (() => {
|
||||
const raw = Number(process.env.GBRAIN_QUEUE_DIVERGENCE_RATIO ?? '');
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 2;
|
||||
})();
|
||||
const divergenceMinWaiting = (() => {
|
||||
const raw = parseInt(process.env.GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING ?? '', 10);
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 50;
|
||||
})();
|
||||
const divergent = stats.by_type.filter(t =>
|
||||
t.waiting_now > divergenceMinWaiting &&
|
||||
t.total > divergenceRatio * Math.max(t.drained_completed, 1));
|
||||
|
||||
// Waiting-TTL cancellations in the window (admission sweep visibility —
|
||||
// derived from the reason prefix cancelJobs writes; no extra storage).
|
||||
let ttlCancelled: Array<{ name: string; count: number }> = [];
|
||||
try {
|
||||
const { TTL_REASON_PREFIX } = await import('../core/minions/admission.ts');
|
||||
const ttlRows = await engine.executeRaw<{ name: string; count: string }>(
|
||||
`SELECT name, count(*)::text AS count FROM minion_jobs
|
||||
WHERE status = 'cancelled' AND error_text LIKE $1
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
GROUP BY name ORDER BY count(*) DESC`,
|
||||
[`${TTL_REASON_PREFIX}%`],
|
||||
);
|
||||
ttlCancelled = ttlRows.map(r => ({ name: r.name, count: parseInt(r.count, 10) }));
|
||||
} catch { /* best-effort */ }
|
||||
// Job names originate from the MCP-exposed submit surface — strip
|
||||
// control/ANSI bytes + cap before echoing into the terminal screams
|
||||
// (same hygiene as frontmatter-derived type names). Names embedded in
|
||||
// COPY-PASTEABLE command hints get the stricter safeConfigSegment gate:
|
||||
// display-sanitize keeps shell metacharacters.
|
||||
const { sanitizeTypeForDisplay: sanitizeName } = await import('../core/schema-pack/type-usage.ts');
|
||||
const { safeConfigSegment } = await import('../core/minions/admission.ts');
|
||||
|
||||
if (hasFlag(args, '--json')) {
|
||||
console.log(JSON.stringify({
|
||||
queue: statsQueue,
|
||||
...stats,
|
||||
divergent: divergent.map(t => ({
|
||||
name: t.name,
|
||||
intake_24h: t.total,
|
||||
drained_completed_24h: t.drained_completed,
|
||||
waiting_now: t.waiting_now,
|
||||
oldest_waiting_minutes: t.oldest_waiting_minutes,
|
||||
})),
|
||||
ttl_cancelled_24h: ttlCancelled,
|
||||
}, null, 2));
|
||||
break;
|
||||
}
|
||||
|
||||
console.log('Job Stats (last 24h):');
|
||||
if (stats.by_type.length > 0) {
|
||||
console.log(` ${'Type'.padEnd(14)} ${'Total'.padEnd(7)} ${'Done'.padEnd(7)} ${'Failed'.padEnd(8)} ${'Dead'.padEnd(6)} Avg Time`);
|
||||
console.log(` ${'Type'.padEnd(14)} ${'Total'.padEnd(7)} ${'Done'.padEnd(7)} ${'Failed'.padEnd(8)} ${'Dead'.padEnd(6)} ${'Drained'.padEnd(9)} ${'Waiting'.padEnd(9)} Avg Time`);
|
||||
for (const t of stats.by_type) {
|
||||
const avgTime = t.avg_duration_ms != null ? `${(t.avg_duration_ms / 1000).toFixed(1)}s` : '—';
|
||||
console.log(` ${t.name.padEnd(14)} ${String(t.total).padEnd(7)} ${String(t.completed).padEnd(7)} ${String(t.failed).padEnd(8)} ${String(t.dead).padEnd(6)} ${avgTime}`);
|
||||
// Drained = terminal outflow in-window, completed-first with the
|
||||
// rest bracketed so TTL-cancel storms can't masquerade as work.
|
||||
const drained = `${t.drained_completed}${(t.drained_failed + t.drained_dead + t.drained_cancelled) > 0 ? `(+${t.drained_failed + t.drained_dead + t.drained_cancelled})` : ''}`;
|
||||
console.log(` ${sanitizeName(t.name).padEnd(14)} ${String(t.total).padEnd(7)} ${String(t.completed).padEnd(7)} ${String(t.failed).padEnd(8)} ${String(t.dead).padEnd(6)} ${drained.padEnd(9)} ${String(t.waiting_now).padEnd(9)} ${avgTime}`);
|
||||
}
|
||||
console.log(` (Drained = completed in-window, +N = failed/dead/cancelled outflow; Waiting = now, all queues)`);
|
||||
} else {
|
||||
console.log(' No jobs in the last 24 hours.');
|
||||
}
|
||||
console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`);
|
||||
|
||||
// DIVERGENT-queue scream: intake structurally exceeds useful drain and a
|
||||
// real backlog is sitting there. This is the default-on protection layer
|
||||
// (quota ships config-only), so it must carry the opt-in hint.
|
||||
for (const t of divergent) {
|
||||
const perDay = t.drained_completed; // window is 24h
|
||||
const etaDays = perDay > 0 ? Math.round(t.waiting_now / perDay) : null;
|
||||
const eta = etaDays != null ? `~${etaDays}d backlog at current drain` : 'backlog never drains at current rate';
|
||||
const ttl = ttlCancelled.find(c => c.name === t.name);
|
||||
const ttlNote = ttl ? ` Waiting-TTL is cancelling ~${ttl.count}/day of it.` : '';
|
||||
console.log(
|
||||
`\n ⚠ DIVERGENT QUEUE type '${sanitizeName(t.name)}': intake ${t.total}/24h vs ${t.drained_completed} completed/24h, ` +
|
||||
`${t.waiting_now} waiting (${eta}).${ttlNote}\n` +
|
||||
` Reduce intake, raise drain, or cap admission:\n` +
|
||||
` gbrain config set minions.quota_max_waiting.${safeConfigSegment(t.name) ?? '<job-name>'} <n>`,
|
||||
);
|
||||
}
|
||||
if (ttlCancelled.length > 0) {
|
||||
const parts = ttlCancelled.map(c => `${sanitizeName(c.name)}: ${c.count}`).join(', ');
|
||||
console.log(
|
||||
`\n ⚠ Waiting-TTL cancelled ${ttlCancelled.reduce((a, c) => a + c.count, 0)} job(s) in the last 24h (${parts}).\n` +
|
||||
` These waited past their TTL without ever being claimed. Tune:\n` +
|
||||
` gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`,
|
||||
);
|
||||
}
|
||||
|
||||
// Scheduling priority (niceness, issue #1815). Best-effort: measures live
|
||||
// workers from the registry + the supervisor (if running) — silently skips
|
||||
// when nothing is reniced/running, so default stats output stays clean.
|
||||
|
||||
@@ -94,15 +94,42 @@ async function runClear(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const force = args.includes('--force');
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
// First non-flag positional after the subcommand is the slug.
|
||||
const slug = args.find((a) => !a.startsWith('--'));
|
||||
const srcIdx = args.indexOf('--source-id');
|
||||
const sourceIdFlag = srcIdx >= 0 && args[srcIdx + 1] && !args[srcIdx + 1].startsWith('--')
|
||||
? args[srcIdx + 1]
|
||||
: undefined;
|
||||
// First non-flag positional after the subcommand is the slug (skip the
|
||||
// --source-id value so it can't be mistaken for the slug).
|
||||
const slug = args.find((a, i) => !a.startsWith('--') && !(srcIdx >= 0 && i === srcIdx + 1));
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain quarantine clear <slug> [--force] [--no-embed]');
|
||||
console.error('Usage: gbrain quarantine clear <slug> [--source-id <id>] [--force] [--no-embed]');
|
||||
process.exit(2);
|
||||
}
|
||||
const page = await engine.getPage(slug);
|
||||
// Deterministic source resolution: an unscoped getPage on a slug that
|
||||
// exists in multiple sources returns an arbitrary row, and the re-import
|
||||
// below writes to WHATEVER source that read happened to hit. Resolve the
|
||||
// candidate sources explicitly; ambiguity is an error, not a coin flip.
|
||||
let sourceId = sourceIdFlag;
|
||||
if (!sourceId) {
|
||||
const rows = await engine.executeRaw<{ source_id: string }>(
|
||||
`SELECT source_id FROM pages WHERE slug = $1 AND deleted_at IS NULL ORDER BY source_id`,
|
||||
[slug],
|
||||
);
|
||||
if (rows.length > 1) {
|
||||
console.error(
|
||||
`Slug "${slug}" exists in ${rows.length} sources: ${rows.map(r => r.source_id).join(', ')}.\n` +
|
||||
`Pick one with: gbrain quarantine clear ${slug} --source-id <id>`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
sourceId = rows[0]?.source_id;
|
||||
}
|
||||
// sourceId is resolved above whenever ANY row exists; zero candidates means
|
||||
// the page doesn't exist in any source, so the 'default' fallback read
|
||||
// returns null and we error below either way.
|
||||
const page = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!page) {
|
||||
console.error(`No page found for slug "${slug}".`);
|
||||
console.error(`No page found for slug "${slug}"${sourceIdFlag ? ` in source "${sourceIdFlag}"` : ''}.`);
|
||||
process.exit(2);
|
||||
}
|
||||
const fm = { ...((page.frontmatter ?? {}) as Record<string, unknown>) };
|
||||
|
||||
@@ -120,6 +120,7 @@ function printCodeModelNudge(decision: Extract<NudgeDecision, { shouldNudge: tru
|
||||
|
||||
interface CodePageRow {
|
||||
slug: string;
|
||||
source_id: string;
|
||||
compiled_truth: string;
|
||||
frontmatter: Record<string, unknown> | null;
|
||||
}
|
||||
@@ -133,8 +134,13 @@ async function fetchCodePages(
|
||||
// Direct SQL: listPages doesn't expose source_id filtering, and we need
|
||||
// compiled_truth + frontmatter anyway (not just the Page shape).
|
||||
const sourceClause = sourceId ? `AND p.source_id = '${sourceId.replace(/'/g, "''")}'` : '';
|
||||
// source_id is SELECTed so the per-page re-import below targets each row's
|
||||
// OWN source. Pre-fix this iterated all sources' code pages but imported
|
||||
// with the CLI-level sourceId (undefined without --source), which — now
|
||||
// that import reads/writes are default-scoped — would duplicate every
|
||||
// non-default-source code page into 'default' and re-embed it.
|
||||
const rows = await engine.executeRaw<CodePageRow>(
|
||||
`SELECT p.slug, p.compiled_truth, p.frontmatter
|
||||
`SELECT p.slug, p.source_id, p.compiled_truth, p.frontmatter
|
||||
FROM pages p
|
||||
WHERE p.type = 'code' ${sourceClause}
|
||||
ORDER BY p.slug
|
||||
@@ -299,7 +305,10 @@ export async function runReindexCode(
|
||||
const result = await importCodeFile(engine, relPath, row.compiled_truth, {
|
||||
noEmbed: opts.noEmbed,
|
||||
force: opts.force,
|
||||
sourceId: opts.sourceId,
|
||||
// Each page re-imports into its OWN source (row-level), not
|
||||
// the CLI-level default — reindex must be an in-place
|
||||
// rebuild, never a cross-source copy.
|
||||
sourceId: row.source_id,
|
||||
});
|
||||
if (result.status === 'imported') reindexed++;
|
||||
else if (result.status === 'skipped') skipped++;
|
||||
|
||||
+157
-5
@@ -6,6 +6,8 @@ import { importFile } from '../core/import-file.ts';
|
||||
import { collectSyncableFiles } from './import.ts';
|
||||
import {
|
||||
isSyncable,
|
||||
isPoisonedPath,
|
||||
sanitizePathForDisplay,
|
||||
unsyncableReason,
|
||||
matchesAnyGlob,
|
||||
resolveSlugForPath,
|
||||
@@ -220,6 +222,19 @@ export interface SyncResult {
|
||||
embedded: number;
|
||||
pagesAffected: string[];
|
||||
failedFiles?: number; // count of parse failures (Bug 9)
|
||||
/**
|
||||
* Files skipped because their FILENAME contains bracket/control characters
|
||||
* (SyncableReason 'malformed-path'). Informational — these never gate
|
||||
* bookmark advancement; rename the files to import them.
|
||||
*/
|
||||
malformedSkipped?: number;
|
||||
/**
|
||||
* Aggregated alias/undeclared explicit-type warnings (schema.type_warnings,
|
||||
* default on) — one entry per distinct non-canonical type this run.
|
||||
* Carried on the RESULT (not just stderr) so worker-driven syncs surface it
|
||||
* in job results where daemon stderr is invisible.
|
||||
*/
|
||||
type_warnings?: Array<{ kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string; count: number }>;
|
||||
/**
|
||||
* v0.41.13.0 partial-sync fields (only set when status === 'partial').
|
||||
*
|
||||
@@ -557,7 +572,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// importFile call below. Codex perf finding #7: per-file loadActivePack adds
|
||||
// disk/YAML/hash overhead × thousands of files. Best-effort: pack load
|
||||
// failure falls through to legacy inferType (parity preserved).
|
||||
let syncActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
|
||||
let syncActivePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> } | undefined;
|
||||
try {
|
||||
// v0.41.37.0 #1569: --no-schema-pack escape hatch. Skip pack load entirely so
|
||||
// no user-supplied pack regex (markdown.ts subtype path_pattern) runs during
|
||||
@@ -1119,18 +1134,43 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// old page's backing file is gone from this source's slice of the repo.
|
||||
const renamedToUnsyncable = manifest.renamed
|
||||
.filter(r => inScope(r.from) && isSyncable(r.from, syncOpts) &&
|
||||
!(inScope(r.to) && isSyncable(r.to, syncOpts)))
|
||||
!(inScope(r.to) && isSyncable(r.to, syncOpts)) &&
|
||||
// A rename onto a NON-poison malformed destination (`foo.md` →
|
||||
// `notes [draft].md`) keeps the old row: the content still exists on
|
||||
// disk under the new name, it just can't re-import until renamed —
|
||||
// deleting the row here would be the rename-lane variant of the
|
||||
// reconcile data-loss class (codex re-review P1). Poisoned
|
||||
// destinations (`](`/control chars) still sweep.
|
||||
!(unsyncableReason(r.to, syncOpts) === 'malformed-path' && !isPoisonedPath(r.to)))
|
||||
.map(r => r.from);
|
||||
const filtered: SyncManifest = {
|
||||
added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
deleted: unique([
|
||||
...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)),
|
||||
// 'malformed-path' deletions MUST still process: the classifier makes
|
||||
// junk filenames unsyncable, but their previously-ingested DB rows are
|
||||
// exactly what a delete event is supposed to remove — filtering them
|
||||
// out here would orphan those rows (searchable forever). Mirror of the
|
||||
// metafile carve-out, in the opposite direction.
|
||||
...manifest.deleted.filter(p => inScope(p) &&
|
||||
(isSyncable(p, syncOpts) || unsyncableReason(p, syncOpts) === 'malformed-path')),
|
||||
...renamedToUnsyncable,
|
||||
]),
|
||||
renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
|
||||
};
|
||||
|
||||
// Surface malformed-filename skips: they were silently dropped from the
|
||||
// `filtered` manifest above, and a skip nobody can see reads as "synced".
|
||||
// Rename DESTINATIONS count too (the rename lane keeps the old row for
|
||||
// non-poison destinations, but the new name still can't import).
|
||||
const malformedSkipped = unique([
|
||||
...[...manifest.added, ...manifest.modified]
|
||||
.filter(p => inScope(p) && unsyncableReason(p, syncOpts) === 'malformed-path'),
|
||||
...manifest.renamed
|
||||
.filter(r => inScope(r.to) && unsyncableReason(r.to, syncOpts) === 'malformed-path')
|
||||
.map(r => r.to),
|
||||
]);
|
||||
|
||||
// NAV-4: warn when --exclude filtered out every candidate change — almost
|
||||
// always a mistyped pattern, and otherwise indistinguishable from
|
||||
// "up to date" in the output.
|
||||
@@ -1155,9 +1195,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (filtered.modified.length) slog(` Modified: ${filtered.modified.join(', ')}`);
|
||||
if (filtered.deleted.length) slog(` Deleted: ${filtered.deleted.join(', ')}`);
|
||||
if (filtered.renamed.length) slog(` Renamed: ${filtered.renamed.map(r => `${r.from} -> ${r.to}`).join(', ')}`);
|
||||
if (malformedSkipped.length) {
|
||||
slog(` Skipped (malformed filename — brackets/control chars; rename to import): ${malformedSkipped.map(sanitizePathForDisplay).join(', ')}`);
|
||||
}
|
||||
if (totalChanges === 0) slog(` No syncable changes.`);
|
||||
return {
|
||||
status: 'dry_run',
|
||||
malformedSkipped: malformedSkipped.length,
|
||||
fromCommit: lastCommit,
|
||||
toCommit: headCommit,
|
||||
added: filtered.added.length,
|
||||
@@ -1207,6 +1251,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// pages every time their materialized file landed in a commit.
|
||||
const reason = unsyncableReason(path, syncOpts);
|
||||
if (reason === 'metafile' || reason === 'pruned-dir') continue;
|
||||
// Bare-bracket markdown (pre-gate imports like `notes [draft].md`) keeps
|
||||
// its row — only the poison signature (`](`/control chars) is sweepable.
|
||||
// Deleting a legit page's row while its file sits on disk is data loss.
|
||||
if (reason === 'malformed-path' && !isPoisonedPath(path)) continue;
|
||||
const slug = await resolveSlugByPathOrSourcePath(engine, path, opts.sourceId);
|
||||
try {
|
||||
const existing = await engine.getPage(slug, pageOpts);
|
||||
@@ -1248,6 +1296,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
|
||||
await clearOpCheckpoint(engine, ckpt.paths);
|
||||
await clearOpCheckpoint(engine, ckpt.target);
|
||||
// A commit whose ONLY changes are malformed filenames lands here with
|
||||
// totalChanges === 0 — the anchor advances past those files forever, so
|
||||
// this early return must surface the skips too (structured-review P2).
|
||||
if (malformedSkipped.length > 0) {
|
||||
serr(
|
||||
` ${malformedSkipped.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
malformedSkipped.map(sanitizePathForDisplay).join(', '),
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: 'up_to_date',
|
||||
fromCommit: lastCommit,
|
||||
@@ -1256,6 +1314,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
chunksCreated: 0,
|
||||
embedded: 0,
|
||||
pagesAffected: [],
|
||||
...(malformedSkipped.length > 0 ? { malformedSkipped: malformedSkipped.length } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1429,6 +1488,23 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// advancement at the bottom of this function.
|
||||
const failedFiles: Array<{ path: string; error: string; line?: number }> = [];
|
||||
|
||||
// Alias-footgun visibility (schema.type_warnings, default on): aggregate
|
||||
// per-file type_warning results ONCE per distinct type per run — an
|
||||
// N-thousand-file sync must warn in O(distinct types) lines, not O(files).
|
||||
const typeWarningCounts = new Map<string, import('../core/schema-pack/type-usage.ts').TypeWarningCount>();
|
||||
const noteTypeWarning = (w: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string } | undefined): void => {
|
||||
if (!w) return;
|
||||
const key = `${w.kind}\t${w.type}`;
|
||||
const cur = typeWarningCounts.get(key);
|
||||
if (cur) cur.count++;
|
||||
else typeWarningCounts.set(key, { ...w, count: 1 });
|
||||
};
|
||||
let typeWarningsEnabled = true;
|
||||
try {
|
||||
const v = await engine.getConfig('schema.type_warnings');
|
||||
typeWarningsEnabled = !(v === 'false' || v === '0' || v === 'off');
|
||||
} catch { /* config unavailable → default on */ }
|
||||
|
||||
// v0.18.0+ multi-source: scope deletePage so we only delete the source-A
|
||||
// row, not every same-slug row across all sources.
|
||||
const deleteOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
@@ -1656,8 +1732,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
try {
|
||||
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
|
||||
importResult = result;
|
||||
noteTypeWarning(result.type_warning);
|
||||
if (result.status === 'imported') chunksCreated += result.chunks;
|
||||
else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
else if (result.status === 'skipped' && result.skip_reason === 'malformed_path') {
|
||||
// Informational skip — a bracket/control-char filename can never
|
||||
// import; counting it as a failure would gate the bookmark forever.
|
||||
serr(` Skipped (malformed filename): ${sanitizePathForDisplay(to)}`);
|
||||
} else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
failedFiles.push({ path: to, error: String((result as { error?: string }).error) });
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
@@ -1922,6 +2003,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// duplicate rows that crashed bare-slug subqueries with Postgres 21000.
|
||||
const result = await observed(pacer, () =>
|
||||
importFile(eng, filePath, path, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack }));
|
||||
noteTypeWarning(result.type_warning);
|
||||
if (result.status === 'imported') {
|
||||
chunksCreated += result.chunks;
|
||||
pagesAffected.push(result.slug);
|
||||
@@ -1935,6 +2017,12 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
filesImported++;
|
||||
// v0.42.x (#1794): checkpoint this path so a kill banks it.
|
||||
await markCompleted(path);
|
||||
} else if (result.status === 'skipped' && result.skip_reason === 'malformed_path') {
|
||||
// Informational skip (bracket/control-char filename): never a
|
||||
// failure, and stable across runs — checkpoint it as done so a
|
||||
// resumed sync doesn't re-attempt it forever.
|
||||
serr(` Skipped (malformed filename — rename to import): ${sanitizePathForDisplay(path)}`);
|
||||
await markCompleted(path);
|
||||
} else if (result.status === 'skipped' && (result as any).error) {
|
||||
failedFiles.push({ path, error: String((result as any).error) });
|
||||
} else {
|
||||
@@ -2444,6 +2532,20 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
slog(`Text imported. Run 'gbrain embed --stale' to generate embeddings.`);
|
||||
}
|
||||
|
||||
if (malformedSkipped.length > 0) {
|
||||
serr(
|
||||
`\n ${malformedSkipped.length} file(s) skipped: malformed filename ` +
|
||||
`(brackets/control chars) — rename to import. Not counted as failures.`,
|
||||
);
|
||||
}
|
||||
|
||||
const typeWarnings = [...typeWarningCounts.values()];
|
||||
if (typeWarningsEnabled && typeWarnings.length > 0) {
|
||||
const { renderTypeWarningSummary } = await import('../core/schema-pack/type-usage.ts');
|
||||
for (const line of renderTypeWarningSummary(typeWarnings)) serr(` ${line}`);
|
||||
serr(` (silence with: gbrain config set schema.type_warnings false)`);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'synced',
|
||||
fromCommit: lastCommit,
|
||||
@@ -2455,6 +2557,8 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
chunksCreated,
|
||||
embedded,
|
||||
pagesAffected,
|
||||
malformedSkipped: malformedSkipped.length,
|
||||
...(typeWarningsEnabled && typeWarnings.length > 0 ? { type_warnings: typeWarnings } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2484,9 +2588,11 @@ async function performFullSync(
|
||||
// code --dry-run` always reported zero files even when ~1500 code
|
||||
// files were waiting.
|
||||
if (opts.dryRun) {
|
||||
const dryRunMalformed: string[] = [];
|
||||
let allFiles = collectSyncableFiles(syncScopeRoot, {
|
||||
strategy: opts.strategy ?? 'markdown',
|
||||
includeGitignored: opts.includeGitignored,
|
||||
onExcluded: (rel) => { dryRunMalformed.push(rel); },
|
||||
});
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
|
||||
@@ -2496,6 +2602,14 @@ async function performFullSync(
|
||||
`${allFiles.length} file(s) would be imported ` +
|
||||
`from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`,
|
||||
);
|
||||
if (dryRunMalformed.length > 0) {
|
||||
slog(
|
||||
` ${dryRunMalformed.length} file(s) would be skipped: malformed filename ` +
|
||||
`(brackets/control chars; rename to import): ` +
|
||||
dryRunMalformed.slice(0, 20).map(sanitizePathForDisplay).join(', ') +
|
||||
(dryRunMalformed.length > 20 ? `, … (+${dryRunMalformed.length - 20} more)` : ''),
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: 'dry_run',
|
||||
fromCommit: null,
|
||||
@@ -2662,10 +2776,23 @@ async function performFullSync(
|
||||
// root-level sync of this source) are out of this walk's sight and must
|
||||
// not be treated as stale.
|
||||
const scopePrefix = slugRoot ? relative(gitContextRoot, syncScopeRoot) + '/' : '';
|
||||
// 'malformed-path' rows ARE reconcile-eligible: junk filenames (bracket /
|
||||
// control-char paths minted by misbehaving producers) can never be
|
||||
// re-imported, so their rows are permanent search pollution unless the
|
||||
// reconcile can sweep them. Strategy safety is preserved by classifier
|
||||
// ordering — a path that fails the strategy check classifies as
|
||||
// 'strategy', never 'malformed-path', so a markdown sync still can't
|
||||
// delete code pages. The #1433 metafile protection is likewise untouched.
|
||||
const reconcileEligible = (p: string): boolean =>
|
||||
isSyncable(p, reconcileSyncOpts) ||
|
||||
// Only the poison signature is sweepable; bare-bracket markdown rows
|
||||
// from pre-gate releases survive reconcile (their file still exists —
|
||||
// deleting the row would be silent data loss; cross-model finding).
|
||||
(unsyncableReason(p, reconcileSyncOpts) === 'malformed-path' && isPoisonedPath(p));
|
||||
const plan = planReconcileDeletes(
|
||||
rows,
|
||||
currentFiles,
|
||||
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && isSyncable(p, reconcileSyncOpts),
|
||||
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && reconcileEligible(p),
|
||||
);
|
||||
if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) {
|
||||
// #2828 mass-delete safety valve: a reconcile that would sweep more than
|
||||
@@ -2722,6 +2849,15 @@ async function performFullSync(
|
||||
);
|
||||
}
|
||||
const deleteScopedOpts = { sourceId: sid };
|
||||
// Malformed-path rows get their own line: unlike genuinely-deleted
|
||||
// files, THEIR backing file is usually still on disk (the walker
|
||||
// excludes it), so "source file was removed" would be a lie and the
|
||||
// rename-to-rescue path must be stated at the moment of removal, not
|
||||
// only in a doctor check the operator may see later (red-team catch).
|
||||
const malformedDeleted = deletableSlugs.filter(slug => {
|
||||
const sp = pathBySlug.get(slug);
|
||||
return sp != null && unsyncableReason(sp, reconcileSyncOpts) === 'malformed-path';
|
||||
}).length;
|
||||
for (let i = 0; i < deletableSlugs.length; i += DELETE_BATCH_SIZE) {
|
||||
const batch = deletableSlugs.slice(i, i + DELETE_BATCH_SIZE);
|
||||
try {
|
||||
@@ -2738,6 +2874,12 @@ async function performFullSync(
|
||||
}
|
||||
if (reconciledDeletes > 0) {
|
||||
slog(` Reconciled ${reconciledDeletes} stale page(s) whose source file was removed.`);
|
||||
if (malformedDeleted > 0) {
|
||||
slog(
|
||||
` (${malformedDeleted} of them had malformed bracket/control-char filenames — ` +
|
||||
`their files may still exist on disk; rename a file to re-import its content.)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2774,6 +2916,11 @@ async function performFullSync(
|
||||
chunksCreated: result.chunksCreated,
|
||||
embedded,
|
||||
pagesAffected: [],
|
||||
// Warning aggregates ride the result for worker/JSON consumers — a full
|
||||
// sync that only prints to a daemon's stderr hides them from cron
|
||||
// topologies (codex re-review; same rationale as the incremental path).
|
||||
...(result.malformedSkipped ? { malformedSkipped: result.malformedSkipped } : {}),
|
||||
...(result.type_warnings ? { type_warnings: result.type_warnings } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3497,6 +3644,11 @@ See also:
|
||||
deleted: r.result.deleted,
|
||||
chunks_created: r.result.chunksCreated,
|
||||
embedded: r.result.embedded,
|
||||
// Warning aggregates (malformed filenames, alias/undeclared
|
||||
// types) — the whole point of the result-field plumbing is that
|
||||
// JSON/worker consumers can see them (codex re-review).
|
||||
...(r.result.malformedSkipped ? { malformed_skipped: r.result.malformedSkipped } : {}),
|
||||
...(r.result.type_warnings ? { type_warnings: r.result.type_warnings } : {}),
|
||||
} : {}),
|
||||
...(r.error ? { error: r.error } : {}),
|
||||
}));
|
||||
|
||||
@@ -463,6 +463,37 @@ export async function runPostUpgrade(args: string[] = []): Promise<void> {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// Waiting-TTL pre-notice (one-shot, warn-before-act). The worker
|
||||
// gates its first sweep behind the SAME flag via runWaitingTtlTick
|
||||
// (notice → grace window → sweep) because daemon restarts never run
|
||||
// this CLI path — this banner is the interactive channel. Stamping
|
||||
// the ISO timestamp here starts the same grace clock, so an operator
|
||||
// who sees this banner gets the full window to tune before anything
|
||||
// is cancelled.
|
||||
try {
|
||||
const { admissionKilled, resolveTtlNames, countTtlExpiredWaiting, ttlNoticeGraceMs, TTL_NOTICE_SHOWN_KEY } =
|
||||
await import('../core/minions/admission.ts');
|
||||
const shown = await engine.getConfig(TTL_NOTICE_SHOWN_KEY);
|
||||
if ((shown == null || shown.trim() === '') && !admissionKilled()) {
|
||||
const ttlNames = await resolveTtlNames(engine);
|
||||
const { total: affected, by_name } = await countTtlExpiredWaiting(engine, ttlNames);
|
||||
const parts = [...ttlNames].map(([name, hours]) => `${name} > ${hours}h: ${by_name[name] ?? 0}`);
|
||||
console.log('');
|
||||
console.log(`⚠ [gbrain] Waiting-TTL is now active: queued jobs that never get claimed are`);
|
||||
console.log(` cancelled after their per-type TTL (${parts.join('; ') || 'defaults'}).`);
|
||||
if (affected > 0) {
|
||||
console.log(` ${affected} currently-queued job(s) already exceed their TTL and will be`);
|
||||
console.log(` cancelled after a ${Math.round(ttlNoticeGraceMs() / 60_000)}min grace window`);
|
||||
console.log(` (auditable error_text; visible in 'gbrain jobs stats').`);
|
||||
}
|
||||
console.log(` Tune or disable: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`);
|
||||
console.log('');
|
||||
await engine.setConfig(TTL_NOTICE_SHOWN_KEY, new Date().toISOString());
|
||||
}
|
||||
} catch {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// #3390: ZeroEntropy sunset notice. ZE announced (2026-07-24) that
|
||||
// its hosted endpoints — including /models/embed and /models/rerank —
|
||||
// shut down on 2026-09-04. Any brain resolving to a zeroentropyai:*
|
||||
|
||||
@@ -30,7 +30,10 @@ export const collectStalledJobs: AdvisorCollector = {
|
||||
severity: 'warn',
|
||||
title: `${r.n} "${r.name}" job${r.n === 1 ? '' : 's'} look stalled (lock lapsed / retrying).`,
|
||||
detail: 'A wedged worker stops backfill/sync from progressing.',
|
||||
fix: { command_argv: ['gbrain', 'jobs', 'status'] },
|
||||
// 'jobs stats' is the real subcommand — 'jobs status' never existed
|
||||
// (the dead fix-command shipped unnoticed because nothing executes
|
||||
// advisor fixes automatically).
|
||||
fix: { command_argv: ['gbrain', 'jobs', 'stats'] },
|
||||
collector: 'stalled-jobs',
|
||||
ask_user: true,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Atomic file write for brain-repo markdown writers.
|
||||
*
|
||||
* Write path: unique tmp sibling → write → fsync → close → (optional verify
|
||||
* of the on-disk bytes) → chmod to the original mode → rename over the target.
|
||||
* The rename is atomic on POSIX filesystems, so readers never observe a torn
|
||||
* file; a crash mid-write leaves only a tmp sibling, never a corrupt target.
|
||||
*
|
||||
* The tmp name embeds pid + random bytes so concurrent writers (two fixers,
|
||||
* a fixer racing a render) can never collide on the tmp path itself. Note the
|
||||
* rename does NOT prevent lost updates between two read-modify-write writers —
|
||||
* callers that need that take the per-page lock (src/core/page-lock.ts).
|
||||
*
|
||||
* Every module used to roll its own copy of this pattern (write-through,
|
||||
* skillopt, schema-pack/mutate, self-upgrade, …). This is the shared home;
|
||||
* migrating the older copies is tracked in TODOS.md.
|
||||
*/
|
||||
|
||||
import {
|
||||
chmodSync,
|
||||
closeSync,
|
||||
existsSync,
|
||||
fsyncSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
statSync,
|
||||
unlinkSync,
|
||||
writeSync,
|
||||
} from 'fs';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { dirname } from 'path';
|
||||
|
||||
export interface AtomicWriteOpts {
|
||||
/**
|
||||
* Called with the bytes read back from the tmp file BEFORE the rename.
|
||||
* Throw to abort the write — the tmp file is removed and the target is
|
||||
* left untouched. Use this to validate that what actually landed on disk
|
||||
* still parses (backlinks uses parseMarkdown here).
|
||||
*/
|
||||
verify?: (onDisk: string) => void;
|
||||
}
|
||||
|
||||
export function atomicWriteFileSync(filePath: string, content: string, opts?: AtomicWriteOpts): void {
|
||||
const tmpPath = `${filePath}.tmp.${process.pid}.${randomBytes(4).toString('hex')}`;
|
||||
|
||||
// Preserve the target's mode across the rename (a fresh tmp file gets the
|
||||
// process umask, which can silently drop e.g. group-write bits).
|
||||
let mode: number | null = null;
|
||||
try {
|
||||
if (existsSync(filePath)) mode = statSync(filePath).mode & 0o7777;
|
||||
} catch {
|
||||
/* stat raced a delete — fall through with default mode */
|
||||
}
|
||||
|
||||
try {
|
||||
const fd = openSync(tmpPath, 'w', mode ?? 0o644);
|
||||
try {
|
||||
// Loop until every byte lands: writeSync may legally return a short
|
||||
// count under disk pressure/quotas, and a silent short write that
|
||||
// truncates AFTER valid frontmatter would pass a frontmatter-only
|
||||
// verifier and atomically install truncated content.
|
||||
const buf = Buffer.from(content, 'utf-8');
|
||||
let off = 0;
|
||||
while (off < buf.length) {
|
||||
const n = writeSync(fd, buf, off, buf.length - off);
|
||||
if (n <= 0) throw new Error(`atomic-write: short write at offset ${off}/${buf.length}`);
|
||||
off += n;
|
||||
}
|
||||
fsyncSync(fd);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
// open(2)'s mode argument is masked by the process umask (0664 & ~022 →
|
||||
// 0644), so an explicit chmod is required to actually PRESERVE the
|
||||
// target's mode across the rename — the pre-wave in-place write kept the
|
||||
// inode's mode exactly; this keeps that property.
|
||||
if (mode !== null) chmodSync(tmpPath, mode);
|
||||
if (opts?.verify) {
|
||||
opts.verify(readFileSync(tmpPath, 'utf-8'));
|
||||
}
|
||||
renameSync(tmpPath, filePath);
|
||||
// Durability of the RENAME itself: fsync the parent directory so a power
|
||||
// loss can't silently drop the new directory entry (the target is never
|
||||
// corrupt either way — this closes the write-vanished window). Dir fsync
|
||||
// is unsupported on some platforms; best-effort by design.
|
||||
try {
|
||||
const dfd = openSync(dirname(filePath), 'r');
|
||||
try { fsyncSync(dfd); } finally { closeSync(dfd); }
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
} catch (err) {
|
||||
try {
|
||||
if (existsSync(tmpPath)) unlinkSync(tmpPath);
|
||||
} catch {
|
||||
/* best-effort cleanup */
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -77,7 +77,7 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'protocol': ['--all', '--allow-empty', '--apply', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--help', '--http', '--image', '--json', '--kind', '--limit', '--max-usd', '--mode', '--multimodal', '--no-embedding', '--offset', '--path', '--progress-interval', '--progress-json', '--quiet', '--save', '--session', '--session-id', '--since', '--slug', '--slugs', '--source', '--stats', '--surface', '--synthesize', '--target', '--timeout', '--token', '--token-ttl', '--trusted-extraction', '--url', '--with-db', '--yes'],
|
||||
'providers': ['--brain', '--ctx-size', '--dim', '--embedding-dimensions', '--embedding-model', '--embeddings', '--fast', '--force', '--from-pages', '--help', '--http', '--json', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--reranking', '--source', '--surface', '--to', '--token-ttl', '--touchpoint', '--version'],
|
||||
'publish': ['--accent', '--bg', '--border', '--brain', '--card-bg', '--code-bg', '--error', '--fg', '--help', '--json', '--link', '--muted', '--out', '--password', '--source', '--title'],
|
||||
'quarantine': ['--aliases', '--all', '--apply', '--brain', '--code', '--compile', '--explain', '--fast', '--fix', '--force', '--force-rechunk', '--from-pages', '--help', '--http', '--include-flagged', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--multimodal', '--near-symbol', '--no-embed', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--restore-only', '--source', '--stale', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl'],
|
||||
'quarantine': ['--aliases', '--all', '--apply', '--brain', '--code', '--compile', '--explain', '--fast', '--fix', '--force', '--force-rechunk', '--from-pages', '--help', '--http', '--include-flagged', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--multimodal', '--near-symbol', '--no-embed', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--restore-only', '--source', '--source-id', '--stale', '--supersessions', '--surface', '--symbol-kind', '--thin', '--token-ttl'],
|
||||
'recall': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--source-guard', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'reconcile-links': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dry-run', '--explain', '--follow', '--help', '--include-frontmatter', '--include-null-signature', '--json', '--name-status', '--no-extract', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--source', '--stale', '--strategy', '--supersessions', '--thin', '--timeout', '--type'],
|
||||
'reindex': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--break-lock', '--code', '--compile', '--concurrency', '--cost-estimate', '--dim', '--dry-run', '--embedding-dimensions', '--embedding-model', '--explain', '--fast', '--fix', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--lang', '--limit', '--markdown', '--max-age', '--max-cost-usd', '--model', '--multimodal', '--no', '--no-embed', '--no-embedding', '--no-extract', '--older-than', '--path', '--pattern', '--pending', '--pglite', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--source', '--stale', '--supabase', '--supersessions', '--surface', '--thin', '--timeout', '--to', '--token-ttl', '--version', '--workers', '--yes'],
|
||||
@@ -106,7 +106,7 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'storage': ['--aliases', '--all', '--brain', '--federated', '--fix', '--help', '--include-null-signature', '--json', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--source-guard', '--stale', '--supersessions', '--thin', '--to'],
|
||||
'sweep': ['--aliases', '--all', '--batch-limit', '--brain', '--budget-ms', '--help', '--include-null-signature', '--json', '--no-extract', '--once', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin', '--timeout'],
|
||||
'sync': ['--abbrev-ref', '--abi', '--abort', '--aliases', '--all', '--all-sources', '--allow-empty', '--asof', '--auto', '--background', '--batch-size', '--brain', '--brain-wide-max-cost-usd', '--branch', '--break-lock', '--by-mention', '--cached', '--catch-up', '--clone-dir', '--code', '--column', '--compile', '--concurrency', '--confirm-destructive', '--content-audit', '--count', '--depth', '--detach', '--diff-filter', '--dim', '--dir', '--drain', '--dry-run', '--embedding-dimensions', '--embedding-model', '--empty', '--exclude', '--exclude-standard', '--explain', '--fast', '--federated', '--ff-only', '--file', '--fix', '--follow', '--force', '--force-break-lock', '--force-rechunk', '--force-retry', '--force-schema', '--format', '--fresh', '--from-meetings', '--from-pages', '--full', '--git-path', '--hard-deadline', '--help', '--http', '--include-flagged', '--include-frontmatter', '--include-gitignored', '--include-null-signature', '--index-audit', '--infer-dates', '--interval', '--is-ancestor', '--json', '--kind', '--lang', '--lock', '--locks', '--markdown', '--max-age', '--max-cost', '--max-cost-usd', '--max-jobs', '--max-rss', '--max-sources', '--max-usd', '--migrate-only', '--missing-path', '--model', '--multimodal', '--name', '--name-only', '--name-status', '--ner', '--nice', '--no-auto-embed', '--no-embed', '--no-embedding', '--no-extract', '--no-federated', '--no-gpg-sign', '--no-hard-deadline', '--no-pull', '--no-recurse-submodules', '--no-renames', '--no-schema-pack', '--no-verify', '--object-format', '--older-than', '--orphan', '--others', '--overwrite', '--pace', '--pace-max-concurrency', '--pack', '--parallel', '--params', '--path', '--pattern', '--pending', '--pglite', '--phase', '--pid-file', '--porcelain', '--prefix', '--priority', '--probe-pglite', '--progress-interval', '--progress-json', '--query', '--queue', '--quiet', '--rebase', '--rebuild-rollup', '--refresh', '--regenerate', '--remediate', '--remediation-plan', '--repo', '--reset', '--resolve', '--respect-gitignore', '--restore-only', '--resume', '--retry-failed', '--run-id', '--save', '--scope', '--serial', '--short', '--show-toplevel', '--since', '--skills-dir', '--skip-failed', '--slug', '--slugs', '--source', '--source-guard', '--source-id', '--src-subpath', '--stale', '--status', '--stdin', '--strategy', '--supabase', '--supersessions', '--surface', '--target', '--target-score', '--thin', '--timeout', '--to', '--token-ttl', '--top-k', '--type', '--url', '--url-managed', '--verbose', '--verify', '--watch', '--window', '--workers', '--yes'],
|
||||
'takes': ['--aliases', '--all', '--brain', '--bucket-size', '--by', '--claim', '--dir', '--domain', '--dry-run', '--evidence', '--expired', '--fast', '--federated', '--force', '--from-pages', '--help', '--holder', '--http', '--include-covered', '--include-null-signature', '--json', '--kind', '--limit', '--max-pages', '--no-embedding', '--no-extract', '--no-federated', '--outcome', '--path', '--pattern', '--pending', '--quality', '--refresh', '--repo', '--reset', '--resolve', '--restore-only', '--row', '--since', '--slugs', '--sort', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--token-ttl', '--unit', '--until', '--value', '--weight', '--who', '--yes'],
|
||||
'takes': ['--aliases', '--all', '--brain', '--bucket-size', '--by', '--claim', '--dir', '--domain', '--dry-run', '--evidence', '--expired', '--fast', '--federated', '--force', '--from-pages', '--help', '--holder', '--http', '--include-covered', '--include-null-signature', '--json', '--kind', '--limit', '--max-pages', '--no-embedding', '--no-extract', '--no-federated', '--outcome', '--path', '--pattern', '--pending', '--quality', '--repo', '--reset', '--resolve', '--restore-only', '--row', '--since', '--slugs', '--sort', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--token-ttl', '--unit', '--until', '--value', '--weight', '--who', '--yes'],
|
||||
'think': ['--aliases', '--all', '--anchor', '--brain', '--calibration-holder', '--explain', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--max-usd', '--mcp-only', '--model', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--rounds', '--save', '--since', '--source', '--stale', '--supersessions', '--surface', '--take', '--thin', '--timeout', '--token-ttl', '--until', '--with-calibration'],
|
||||
'transcripts': ['--aliases', '--all', '--all-discovery', '--background', '--brain', '--brain-wide-max-cost-usd', '--by-mention', '--code', '--compile', '--days', '--dry-run', '--embed', '--explain', '--facts', '--fast', '--federated', '--follow', '--force', '--format', '--from-pages', '--full', '--help', '--http', '--include-null-signature', '--json', '--limit', '--markdown', '--max-cost-usd', '--no-embedding', '--no-extract', '--no-federated', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--repo', '--reset', '--resolve', '--restore-only', '--since', '--slug', '--source', '--source-guard', '--source-id', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl'],
|
||||
'upgrade': ['--all', '--apply-clean-hunks', '--brain', '--check', '--code', '--compile', '--concurrency', '--detail', '--dim', '--embedding-dimensions', '--embedding-model', '--fast', '--ff-only', '--flag', '--force', '--force-all', '--force-orchestrator', '--force-retry', '--force-schema', '--format', '--from-pages', '--help', '--host-dir', '--http', '--inject-bootstrap', '--inline', '--install', '--interval', '--json', '--limit', '--list', '--markdown', '--max-rss', '--migrate-only', '--migration', '--mode', '--model', '--multimodal', '--name-only', '--no', '--no-autopilot-install', '--no-embed', '--no-embedding', '--no-inject', '--no-worker', '--non-interactive', '--now', '--path', '--pglite', '--quiet', '--repo', '--reset', '--since', '--skills-dir', '--skip-verify', '--source', '--stale', '--status', '--supabase', '--surface', '--swap-only', '--target', '--to', '--token-ttl', '--uninstall', '--user', '--verbose', '--verify', '--version', '--workers', '--yes'],
|
||||
|
||||
@@ -1191,6 +1191,10 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
|
||||
// stops claiming "Nothing in gbrain reads this" for a key the resolver
|
||||
// reads on every unqualified call.
|
||||
'sources.default',
|
||||
// Alias/undeclared explicit-type warnings at sync/import (default on).
|
||||
// Read by performSync + runImport summary aggregation; 'false'/'0'/'off'
|
||||
// silences both surfaces (schema lint rules stay active).
|
||||
'schema.type_warnings',
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -1211,6 +1215,12 @@ export const KNOWN_CONFIG_KEY_PREFIXES: readonly string[] = [
|
||||
'autopilot.', // autopilot.nightly_quality_probe.*, autopilot.auto_drain.* (#1685)
|
||||
'chronicle.', // chronicle.tz + future Life Chronicle knobs (#2390)
|
||||
'self_upgrade.', // v0.42 self-upgrade (mode, quiet_hours, state)
|
||||
// Queue admission control (per-name sub-keys):
|
||||
// minions.coalesce_params.<name>, minions.ttl_waiting_hours.<name>,
|
||||
// minions.quota_max_waiting.<name>, plus the one-time
|
||||
// minions.ttl_notice_shown flag. Booleans via the canonical truthiness
|
||||
// parser; numeric 0 disables.
|
||||
'minions.',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -24,6 +24,7 @@ import { randomUUID } from 'node:crypto';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { PhaseResult, PhaseError } from '../cycle.ts';
|
||||
import { MinionQueue } from '../minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../minions/admission.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, MinionJobStatus, SubagentHandlerData } from '../minions/types.ts';
|
||||
import { serializeMarkdown } from '../markdown.ts';
|
||||
@@ -211,9 +212,20 @@ export async function runPhasePatterns(
|
||||
timeout_ms: budgets.timeoutMs,
|
||||
queue: childQueueName,
|
||||
};
|
||||
const job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
let job: Awaited<ReturnType<typeof queue.add>>;
|
||||
try {
|
||||
job = await queue.add('subagent', data as unknown as Record<string, unknown>, submitOpts, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only): a
|
||||
// rejected submit is a recorded phase SKIP, never a phase crash — the
|
||||
// next cycle retries once the backlog drains.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
return skipped('admission_quota', e.message);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Drain this phase's private child queue inline so the parent observes
|
||||
// the terminal state instead of polling waitForCompletion until
|
||||
|
||||
@@ -52,6 +52,7 @@ import { parseLlmJson } from '../llm-json.ts';
|
||||
import type { BrainEngine, DreamVerdict, TriageSegment } from '../engine.ts';
|
||||
import type { PhaseResult, PhaseError } from '../cycle.ts';
|
||||
import { MinionQueue } from '../minions/queue.ts';
|
||||
import { isQueueQuotaExceededError } from '../minions/admission.ts';
|
||||
import { reconnectAfterConnectionError } from '../minions/reconnect.ts';
|
||||
import { isRetryableConnError } from '../retry-matcher.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
@@ -821,7 +822,15 @@ export async function runPhaseSynthesize(
|
||||
}
|
||||
}
|
||||
|
||||
// Admission-quota latch: once a submit is rejected, every later transcript
|
||||
// this run would be rejected too — record one skip per remaining file
|
||||
// without hammering the queue.
|
||||
let quotaHit = false;
|
||||
for (const t of worthProcessing) {
|
||||
if (quotaHit) {
|
||||
skipReports.push({ filePath: t.filePath, reason: 'admission_quota: submission stopped this run' });
|
||||
continue;
|
||||
}
|
||||
const hash16 = t.contentHash.slice(0, 16);
|
||||
const hash6 = t.contentHash.slice(0, 6);
|
||||
|
||||
@@ -911,6 +920,12 @@ export async function runPhaseSynthesize(
|
||||
? `anthropic:${config.model}`
|
||||
: config.model;
|
||||
const triageVerdict = pass.byPath.get(t.filePath);
|
||||
// Fresh (non-coalesced) chunk submissions for THIS transcript — rolled
|
||||
// back if a later chunk hits the admission quota, so a transcript never
|
||||
// half-synthesizes while its skip report claims it was skipped
|
||||
// (adversarial finding). Coalesced rows are another run's bookkeeping
|
||||
// and must not be cancelled.
|
||||
const transcriptFreshIds: number[] = [];
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const childData: SubagentHandlerData = {
|
||||
prompt: buildSynthesisPrompt(
|
||||
@@ -940,12 +955,36 @@ export async function runPhaseSynthesize(
|
||||
timeout_ms: config.subagentTimeoutMs,
|
||||
queue: childQueueName,
|
||||
};
|
||||
let child = await queue.add(
|
||||
'subagent',
|
||||
childData as unknown as Record<string, unknown>,
|
||||
submitOpts,
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
let child: Awaited<ReturnType<typeof queue.add>>;
|
||||
try {
|
||||
child = await queue.add(
|
||||
'subagent',
|
||||
childData as unknown as Record<string, unknown>,
|
||||
submitOpts,
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only):
|
||||
// a rejected submit is a recorded phase skip, never a phase crash —
|
||||
// same posture as daily_cap_reached. The quota won't clear mid-run,
|
||||
// so stop submitting for this run entirely. Roll back this
|
||||
// transcript's already-submitted fresh chunks first: draining a
|
||||
// partial chunk set would write partial pages for a transcript the
|
||||
// skip report says was skipped.
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
for (const id of transcriptFreshIds) {
|
||||
try { await queue.cancelJob(id); } catch { /* best-effort rollback */ }
|
||||
const idx = childIds.indexOf(id);
|
||||
if (idx >= 0) childIds.splice(idx, 1);
|
||||
jobRawSource.delete(id);
|
||||
chunkInfo.delete(id);
|
||||
}
|
||||
skipReports.push({ filePath: t.filePath, reason: `admission_quota: ${e.message}` });
|
||||
quotaHit = true;
|
||||
break;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
// Self-heal (#4152 C1): an idempotency-coalesced row still `waiting`
|
||||
// in a FOREIGN dream-inline-* queue was stranded by a previously
|
||||
// killed/timed-out run — no worker will ever claim it, and waiting on
|
||||
@@ -971,7 +1010,10 @@ export async function runPhaseSynthesize(
|
||||
);
|
||||
}
|
||||
}
|
||||
if (child.coalesced !== true) submittedToday++;
|
||||
if (child.coalesced !== true) {
|
||||
submittedToday++;
|
||||
transcriptFreshIds.push(child.id);
|
||||
}
|
||||
childIds.push(child.id);
|
||||
jobRawSource.set(child.id, t.filePath);
|
||||
if (isChunked) {
|
||||
|
||||
@@ -86,6 +86,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'facts_extraction_health',
|
||||
'facts_health',
|
||||
'frontmatter_integrity',
|
||||
'malformed_path_pages',
|
||||
'grade_confidence_drift',
|
||||
'graph_coverage',
|
||||
'graph_signals_coverage',
|
||||
|
||||
+82
-12
@@ -4,12 +4,13 @@ import { createHash } from 'crypto';
|
||||
import { marked } from 'marked';
|
||||
import type { BrainEngine, FileSpec } from './engine.ts';
|
||||
import { parseMarkdown } from './markdown.ts';
|
||||
import { classifyStoredType } from './schema-pack/type-usage.ts';
|
||||
import { chunkText } from './chunkers/recursive.ts';
|
||||
import { chunkCodeText, chunkCodeTextFull, detectCodeLanguage, CHUNKER_VERSION } from './chunkers/code.ts';
|
||||
import { findChunkForOffset } from './chunkers/edge-extractor.ts';
|
||||
import { extractCodeRefs, imageOfCandidates } from './link-extraction.ts';
|
||||
import { embedBatch, embedMultimodal, currentEmbeddingSignature } from './embedding.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath, hasMalformedPathSegment } from './sync.ts';
|
||||
import type { ChunkInput, PageInput, PageType } from './types.ts';
|
||||
import { computeEffectiveDate } from './effective-date.ts';
|
||||
import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts';
|
||||
@@ -235,6 +236,21 @@ export interface ImportResult {
|
||||
flagged?: boolean;
|
||||
/** Which flag tier fired, when `flagged`. */
|
||||
flag_reason?: 'markup_heavy' | 'oversized';
|
||||
/**
|
||||
* Machine-readable skip class for status='skipped' rows that must NOT be
|
||||
* treated as failures. 'malformed_path' = the FILENAME contains bracket or
|
||||
* control characters (never importable; rename the file) — sync counts these
|
||||
* in its malformed summary and keeps them OUT of failedFiles / the failure
|
||||
* ledger so they can never gate bookmark advancement.
|
||||
*/
|
||||
skip_reason?: 'malformed_path';
|
||||
/**
|
||||
* Advisory (schema.type_warnings): the page's explicit frontmatter `type:`
|
||||
* is an alias of a canonical pack type or undeclared in the pack. The type
|
||||
* is stored literally either way; sync/import aggregate these once per
|
||||
* distinct type per run.
|
||||
*/
|
||||
type_warning?: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string };
|
||||
}
|
||||
|
||||
const MAX_FILE_SIZE = 5_000_000; // 5MB
|
||||
@@ -295,7 +311,7 @@ export async function importFromContent(
|
||||
* Callers thread this from `loadActivePack(ctx)` once per command —
|
||||
* NEVER per file inside sync (codex perf finding #7).
|
||||
*/
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> };
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
|
||||
/**
|
||||
* v0.39.3.0 provenance write-through (WARN-8). When set, threaded to
|
||||
* `tx.putPage` so the page's `source_kind`, `source_uri`,
|
||||
@@ -583,7 +599,11 @@ export async function importFromContent(
|
||||
// #1035: fetch the existing page BEFORE the hash compute so (a) the type
|
||||
// preservation below participates in the hash (a no-op re-put stays a
|
||||
// hash-match skip) and (b) the hash short-circuit below reuses this row.
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
// Scoped to the exact (source_id, slug) row the writes below target —
|
||||
// engine.putPage defaults to 'default' when sourceId is unset, so the read
|
||||
// mirrors that default instead of matching the slug in ANY source (the
|
||||
// unscoped-check/scoped-write bug class).
|
||||
const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
|
||||
// #2044: remote get_page intentionally strips private facts rows. A
|
||||
// documented get_page -> edit -> put_page round-trip can therefore arrive
|
||||
@@ -614,6 +634,22 @@ export async function importFromContent(
|
||||
parsed.type = existing.type;
|
||||
}
|
||||
|
||||
// Alias-footgun visibility: an explicit frontmatter `type:` that is an
|
||||
// ALIAS of a canonical pack type (or entirely undeclared) is stored
|
||||
// literally and never re-normalized — different agents can silently file
|
||||
// the same concept under different types/directories. Classify it here
|
||||
// (once per file, aggregated once per type per run by sync/import) so the
|
||||
// misroute class is loud. Purely advisory: the type is still stored as-is.
|
||||
let typeWarning: ImportResult['type_warning'];
|
||||
if (parsed.typeExplicit === true && opts.activePack) {
|
||||
const cls = classifyStoredType(parsed.type, opts.activePack);
|
||||
if (cls.kind === 'alias_of') {
|
||||
typeWarning = { kind: 'alias_of', type: parsed.type, canonical: cls.canonical, directory: cls.directory };
|
||||
} else if (cls.kind === 'undeclared') {
|
||||
typeWarning = { kind: 'undeclared', type: parsed.type };
|
||||
}
|
||||
}
|
||||
|
||||
const HASH_EPHEMERAL_FRONTMATTER_KEYS = [
|
||||
'captured_at',
|
||||
'ingested_at',
|
||||
@@ -647,7 +683,7 @@ export async function importFromContent(
|
||||
};
|
||||
|
||||
if (existing?.content_hash === hash && !opts.forceRechunk) {
|
||||
return { slug, status: 'skipped', chunks: 0, parsedPage };
|
||||
return { slug, status: 'skipped', chunks: 0, parsedPage, ...(typeWarning ? { type_warning: typeWarning } : {}) };
|
||||
}
|
||||
|
||||
// v0.41.13 (#1309) — identity-based cross-slug dedup pre-check.
|
||||
@@ -689,7 +725,7 @@ export async function importFromContent(
|
||||
}
|
||||
if (dup && dup.slug !== slug) {
|
||||
// Look up the duplicate page so we can compare frontmatter.id.
|
||||
const dupPage = await engine.getPage(dup.slug, sourceId ? { sourceId } : undefined);
|
||||
const dupPage = await engine.getPage(dup.slug, { sourceId: sourceId ?? 'default' });
|
||||
const dupFmId = (dupPage?.frontmatter as Record<string, unknown> | undefined)?.id;
|
||||
const dupFmIdStr = typeof dupFmId === 'string' && dupFmId.length > 0 ? dupFmId : null;
|
||||
const sameExternalId = fmIdStr !== null && dupFmIdStr === fmIdStr;
|
||||
@@ -820,7 +856,7 @@ export async function importFromContent(
|
||||
// caller's sourceId so writes target (sourceId, slug) rather than the
|
||||
// schema DEFAULT — required for multi-source brains; harmless ('default')
|
||||
// for single-source callers.
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
const txOpts = { sourceId: sourceId ?? 'default' };
|
||||
await engine.transaction(async (tx) => {
|
||||
if (existing) await tx.createVersion(slug, txOpts);
|
||||
|
||||
@@ -1000,6 +1036,7 @@ export async function importFromContent(
|
||||
parsedPage,
|
||||
...(pageQuarantined ? { quarantined: true } : {}),
|
||||
...(pageFlagged ? { flagged: true, flag_reason: pageFlagReason } : {}),
|
||||
...(typeWarning ? { type_warning: typeWarning } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1021,7 +1058,7 @@ async function verifyPageReadable(
|
||||
sourceId: string | undefined,
|
||||
caller: string,
|
||||
): Promise<void> {
|
||||
const readBack = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
const readBack = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!readBack) {
|
||||
// Log to ingest_log before throwing so the failure is durable and
|
||||
// agent-inspectable, not just a transient stderr message.
|
||||
@@ -1087,7 +1124,7 @@ export async function importFromFile(
|
||||
* `parseMarkdown` uses pack-driven type inference. Load ONCE per command;
|
||||
* never per file (codex perf finding #7).
|
||||
*/
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> };
|
||||
activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
|
||||
} = {},
|
||||
): Promise<ImportResult> {
|
||||
// Defense-in-depth: reject symlinks before reading content.
|
||||
@@ -1103,6 +1140,27 @@ export async function importFromFile(
|
||||
|
||||
let content = readFileSync(filePath, 'utf-8');
|
||||
|
||||
// Defense-in-depth for callers that bypass the sync/import classifiers
|
||||
// (direct importFromFile, reindex, capture paths): a malformed filename is
|
||||
// never importable. Checked BEFORE the code dispatch and BEFORE any YAML
|
||||
// parsing (codex re-review P2: a control-char code path returned through
|
||||
// importCodeFile, and broken-YAML junk returned a parse error instead of
|
||||
// this informational skip). hasMalformedPathSegment is markdown-scoped for
|
||||
// brackets, so legit bracketed code dirs (`app/[id]/`) still dispatch;
|
||||
// control characters reject on every path. skip_reason marks this as
|
||||
// informational so sync's failure gate never counts it.
|
||||
if (hasMalformedPathSegment(relativePath)) {
|
||||
return {
|
||||
slug: '',
|
||||
status: 'skipped',
|
||||
skip_reason: 'malformed_path',
|
||||
chunks: 0,
|
||||
error:
|
||||
`Path "${relativePath}" contains bracket or control characters and ` +
|
||||
`cannot be imported. Rename the file to import it.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Route code files through the code import path
|
||||
if (isCodeFilePath(relativePath)) {
|
||||
return importCodeFile(engine, relativePath, content, {
|
||||
@@ -1149,6 +1207,10 @@ export async function importFromFile(
|
||||
// parsed.slug is `frontmatter.slug || inferSlug(filePath)` where inferSlug
|
||||
// falls back to slugifyPath(). So parsed.slug.length > 0 with empty
|
||||
// expectedSlug = frontmatter provided one; both empty = no usable slug.
|
||||
// (The malformed-path defense runs earlier, before the code dispatch —
|
||||
// slugifyPath must never see a junk filename: it would STRIP the brackets
|
||||
// and mint a plausible-looking slug, the exact mechanism that polluted
|
||||
// search in the poisoned-path incident.)
|
||||
const expectedSlug = slugifyPath(relativePath);
|
||||
let resolvedSlug = expectedSlug;
|
||||
let usedFrontmatterFallback = false;
|
||||
@@ -1246,7 +1308,7 @@ export async function importCodeFile(
|
||||
const lang = detectCodeLanguage(relativePath) || 'unknown';
|
||||
const title = `${relativePath} (${lang})`;
|
||||
const sourceId = opts.sourceId;
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
const txOpts = { sourceId: sourceId ?? 'default' };
|
||||
// PostgreSQL text columns reject U+0000 even though source files may
|
||||
// legitimately contain it inside string/regex fixtures. Preserve a visible,
|
||||
// searchable representation instead of dropping the entire code page.
|
||||
@@ -1279,7 +1341,11 @@ export async function importCodeFile(
|
||||
.update(JSON.stringify({ title, type: 'code', content, lang, chunker_version: CHUNKER_VERSION }))
|
||||
.digest('hex');
|
||||
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
// Scoped to the exact (source_id, slug) row the writes below target —
|
||||
// engine.putPage defaults to 'default' when sourceId is unset, so the read
|
||||
// mirrors that default instead of matching the slug in ANY source (the
|
||||
// unscoped-check/scoped-write bug class).
|
||||
const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!opts.force && existing?.content_hash === hash) {
|
||||
return { slug, status: 'skipped', chunks: 0 };
|
||||
}
|
||||
@@ -1317,7 +1383,7 @@ export async function importCodeFile(
|
||||
// OpenAI API. Order matters: our chunk_index is semantic (tree-sitter
|
||||
// order), so a matching (chunk_index, text_hash) means a verbatim
|
||||
// preserved symbol.
|
||||
const existingChunks = existing ? await engine.getChunks(slug, sourceId ? { sourceId } : undefined) : [];
|
||||
const existingChunks = existing ? await engine.getChunks(slug, { sourceId: sourceId ?? 'default' }) : [];
|
||||
const existingByKey = new Map<string, typeof existingChunks[number]>();
|
||||
for (const ec of existingChunks) {
|
||||
existingByKey.set(`${ec.chunk_index}:${ec.chunk_text}`, ec);
|
||||
@@ -1775,7 +1841,11 @@ export async function importImageFile(
|
||||
// and slugifyPath would already preserve it). Recompute with the file
|
||||
// extension preserved so the page slug is stable + collision-free.
|
||||
const imageSlug = relativePath.replace(/[\\\/]/g, '/').toLowerCase();
|
||||
const sourceOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
// Scoped to the exact (source_id, slug) row the write targets — same
|
||||
// unscoped-check/scoped-write fix as importFromContent/importCodeFile
|
||||
// above (the variable-bound ternary shape evaded the CI guard's inline
|
||||
// heuristic; caught by adversarial review).
|
||||
const sourceOpts = { sourceId: opts.sourceId ?? 'default' };
|
||||
const linkOpts = opts.sourceId
|
||||
? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId, originSourceId: opts.sourceId }
|
||||
: undefined;
|
||||
|
||||
@@ -1053,7 +1053,9 @@ export function makeResolver(
|
||||
// (unwrapped by unwrapWikilink) that name a real page the strict regex
|
||||
// could not reach and whose full-path fuzzy score is below threshold.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) {
|
||||
const page = await engine.getPage(trimmed);
|
||||
// Same source scope as the basename index above (#972): a wikilink in
|
||||
// source A must not resolve to a same-slug page in source B.
|
||||
const page = await engine.getPage(trimmed, opts.sourceId ? { sourceId: opts.sourceId } : undefined); // gbrain-allow-unscoped-getpage: read-only wikilink resolution; unscoped-when-no-source is the documented single-source behavior
|
||||
if (page) {
|
||||
cache.set(cacheKey, trimmed);
|
||||
return trimmed;
|
||||
@@ -1065,7 +1067,7 @@ export function makeResolver(
|
||||
for (const hint of hints) {
|
||||
if (!hint) continue;
|
||||
const candidate = `${hint}/${slugified}`;
|
||||
const page = await engine.getPage(candidate);
|
||||
const page = await engine.getPage(candidate, opts.sourceId ? { sourceId: opts.sourceId } : undefined); // gbrain-allow-unscoped-getpage: read-only wikilink resolution; unscoped-when-no-source is the documented single-source behavior
|
||||
if (page) {
|
||||
cache.set(cacheKey, candidate);
|
||||
return candidate;
|
||||
|
||||
@@ -76,6 +76,45 @@ export function coerceFrontmatterString(v: unknown): string {
|
||||
return String(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte offset of the first character AFTER the closing frontmatter fence —
|
||||
* i.e. where the body starts and where a body-only editor may safely operate
|
||||
* without ever touching frontmatter bytes.
|
||||
*
|
||||
* Fence semantics mirror collectValidationErrors exactly (the canonical
|
||||
* definition): leading blank lines are allowed before the opener, fences are
|
||||
* matched with trim() so CRLF line endings (`---\r`) count. Returns 0 when the
|
||||
* file has no frontmatter at all (first non-empty line is not `---`) — there
|
||||
* is no fence to protect, the whole file is body. Returns 0 for an UNCLOSED
|
||||
* fence too; callers that must not edit such files should pre-validate with
|
||||
* parseMarkdown({validate:true}) and treat MISSING_CLOSE as a blocker (the
|
||||
* backlinks fixer does).
|
||||
*/
|
||||
export function frontmatterBodyOffset(content: string): number {
|
||||
const lines = content.split('\n');
|
||||
|
||||
let offset = 0;
|
||||
let i = 0;
|
||||
// Skip leading blank lines.
|
||||
for (; i < lines.length; i++) {
|
||||
if (lines[i].trim().length > 0) break;
|
||||
offset += lines[i].length + 1;
|
||||
}
|
||||
if (i >= lines.length) return 0; // empty / whitespace-only file
|
||||
if (lines[i].trim() !== '---') return 0; // no frontmatter
|
||||
|
||||
offset += lines[i].length + 1; // consume the opening fence line
|
||||
for (i = i + 1; i < lines.length; i++) {
|
||||
const isLast = i === lines.length - 1;
|
||||
const lineLen = lines[i].length + (isLast ? 0 : 1);
|
||||
offset += lineLen;
|
||||
if (lines[i].trim() === '---') {
|
||||
return Math.min(offset, content.length);
|
||||
}
|
||||
}
|
||||
return 0; // unclosed fence — no safe body offset
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a markdown file with YAML frontmatter into its components.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
/**
|
||||
* Admission control for the minion queue — the submit-side half of the
|
||||
* queue-divergence fix (the drain-side pool-starvation half landed in
|
||||
* v0.46.1.0). Three primitives, all ADMISSION-side by design (claim
|
||||
* fairness / lane scheduling is explicitly out of scope, tracked in TODOS):
|
||||
*
|
||||
* 1. PARAM-COALESCING — an identical parentless submit (same name, queue,
|
||||
* owner lane, and payload hash) coalesces onto the newest matching
|
||||
* WAITING row instead of enqueuing a duplicate. Targets the
|
||||
* runaway-producer class: crons re-submitting the same prompt hundreds
|
||||
* of times a day into a queue that drains a fraction of that.
|
||||
* 2. WAITING-TTL — a job still WAITING after N hours is cancelled (via the
|
||||
* canonical cancel path, so parents/aggregators resolve) with an
|
||||
* auditable error_text. At structural divergence (intake >> drain),
|
||||
* FIFO wait exceeds any plausible usefulness horizon — cancelling
|
||||
* visibly beats queueing forever.
|
||||
* 3. NAME-GLOBAL QUOTA — reject (typed error, never a silent coalesce)
|
||||
* submits once a name's TOTAL waiting count across ALL queues reaches
|
||||
* the configured cap. Counts name-globally because fanout producers use
|
||||
* per-run private queues (dream-inline-*): a (name, queue)-scoped count
|
||||
* would reset to zero for every new private queue and never bind.
|
||||
* NO shipped default (user decision D2C) — activates only via config.
|
||||
*
|
||||
* Per-name defaults table pattern follows handler-timeouts.ts. Config keys
|
||||
* (DB plane, registered under the 'minions.' prefix):
|
||||
* minions.coalesce_params.<name> ('false'/'0'/'off' disables; default on
|
||||
* only for names in PARAM_COALESCE_DEFAULT)
|
||||
* minions.ttl_waiting_hours.<name> (number; 0 disables; default only for
|
||||
* names in WAITING_TTL_DEFAULT_HOURS)
|
||||
* minions.quota_max_waiting.<name> (number; no defaults)
|
||||
*
|
||||
* Env kill-switch: GBRAIN_MINIONS_ADMISSION=0 disables all three wholesale
|
||||
* (incident escape hatch — no DB needed).
|
||||
*
|
||||
* All lookups fail OPEN to the defaults tables: an unreadable config must
|
||||
* never block job submission. The first failure per process logs one stderr
|
||||
* warning (a silent fail-open is a silent failure).
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
|
||||
/** Names whose parentless submits coalesce on identical params by default. */
|
||||
export const PARAM_COALESCE_DEFAULT: Readonly<Record<string, boolean>> = {
|
||||
subagent: true,
|
||||
};
|
||||
|
||||
/** Default waiting-TTL hours per name. Absent = no TTL. */
|
||||
export const WAITING_TTL_DEFAULT_HOURS: Readonly<Record<string, number>> = {
|
||||
subagent: 48,
|
||||
};
|
||||
|
||||
/**
|
||||
* Default name-global waiting quotas. EMPTY by design (user decision D2C):
|
||||
* the quota mechanism ships but activates only via
|
||||
* `minions.quota_max_waiting.<name>` config. The DIVERGENT-queue scream in
|
||||
* `jobs stats` / doctor is the default-on protection layer and carries the
|
||||
* opt-in hint.
|
||||
*/
|
||||
export const QUOTA_MAX_WAITING_DEFAULT: Readonly<Record<string, number>> = {};
|
||||
|
||||
/**
|
||||
* Keys excluded from the param hash. ONLY the hash's own storage key:
|
||||
* `__owner_client_id` is deliberately INCLUDED so coalescing never crosses
|
||||
* owner lanes — one OAuth client's submit must not be suppressed by (or
|
||||
* handed a job id owned by) another client.
|
||||
*/
|
||||
export const PARAM_HASH_EXCLUDED_KEYS: ReadonlySet<string> = new Set(['__param_hash']);
|
||||
|
||||
/**
|
||||
* error_text prefix stamped by the waiting-TTL sweep. The 24h-cancellation
|
||||
* surfaces (jobs stats, doctor) LIKE-match on this exact prefix — keep the
|
||||
* sweep's reason string and the consumers' patterns derived from ONE constant
|
||||
* so they can never drift apart.
|
||||
*/
|
||||
export const TTL_REASON_PREFIX = 'waiting_ttl_expired';
|
||||
|
||||
/**
|
||||
* Config flag recording WHEN the one-time waiting-TTL notice was shown
|
||||
* (ISO timestamp; legacy value 'true' = shown at unknown time). Shared by
|
||||
* the worker's warn-before-act gate and the runPostUpgrade banner.
|
||||
*/
|
||||
export const TTL_NOTICE_SHOWN_KEY = 'minions.ttl_notice_shown';
|
||||
|
||||
/**
|
||||
* Grace window between the one-time TTL notice and the first sweep (user
|
||||
* requirement D1A: warn BEFORE acting, with enough time to actually react —
|
||||
* one maintenance tick (~30s) is not a warning, it's a courtesy log line).
|
||||
* Env override is a test seam / incident hatch.
|
||||
*/
|
||||
export const TTL_NOTICE_GRACE_MS_DEFAULT = 60 * 60 * 1000;
|
||||
|
||||
export function ttlNoticeGraceMs(): number {
|
||||
const v = Number(process.env.GBRAIN_MINIONS_TTL_NOTICE_GRACE_MS);
|
||||
return Number.isFinite(v) && v >= 0 ? v : TTL_NOTICE_GRACE_MS_DEFAULT;
|
||||
}
|
||||
|
||||
/** Typed admission rejection — submitters surface the message or record a skip. */
|
||||
export class QueueQuotaExceededError extends Error {
|
||||
readonly code = 'quota_exceeded';
|
||||
constructor(
|
||||
public readonly jobName: string,
|
||||
public readonly waiting: number,
|
||||
public readonly quota: number,
|
||||
) {
|
||||
// The message travels to REMOTE MCP clients via submit_agent — it names
|
||||
// the quota (the caller's admission contract) but NOT the live global
|
||||
// waiting count, which would leak cross-tenant queue depth. Operators get
|
||||
// exact counts locally from 'gbrain jobs stats'; the count stays on the
|
||||
// error object for local consumers/tests.
|
||||
super(
|
||||
`queue admission: '${jobName}' is at its waiting quota (${quota}, all queues). ` +
|
||||
`Drain or cancel backlog first — see 'gbrain jobs stats'. ` +
|
||||
`Tune: gbrain config set minions.quota_max_waiting.${jobName} <n> (raise) or remove the key (disable).`,
|
||||
);
|
||||
this.name = 'QueueQuotaExceededError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical quota-rejection check for submitters (agent fanout, dream
|
||||
* synthesize/patterns, ops). Checks the stable `code` field as well as
|
||||
* instanceof/name so it survives dual-module instances and compiled-binary
|
||||
* boundaries where instanceof can lie.
|
||||
*/
|
||||
export function isQueueQuotaExceededError(e: unknown): e is QueueQuotaExceededError {
|
||||
if (e instanceof QueueQuotaExceededError) return true;
|
||||
if (!(e instanceof Error)) return false;
|
||||
return e.name === 'QueueQuotaExceededError' ||
|
||||
(e as { code?: unknown }).code === 'quota_exceeded';
|
||||
}
|
||||
|
||||
/** Stable stringify: recursively sorts object keys so hash(key order) is invariant. */
|
||||
function stableStringify(v: unknown): string {
|
||||
if (v === null || typeof v !== 'object') return JSON.stringify(v) ?? 'undefined';
|
||||
if (Array.isArray(v)) return `[${v.map(stableStringify).join(',')}]`;
|
||||
const o = v as Record<string, unknown>;
|
||||
const keys = Object.keys(o).sort();
|
||||
return `{${keys.map(k => `${JSON.stringify(k)}:${stableStringify(o[k])}`).join(',')}}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* sha256 over the stable-stringified payload minus PARAM_HASH_EXCLUDED_KEYS.
|
||||
* Node-side (not SQL md5(data::text)) so canonicalization is explicit and
|
||||
* unit-testable, with no reliance on jsonb text-rendering parity across
|
||||
* engines. Stored in data.__param_hash (the `__`-prefixed embedded-metadata
|
||||
* convention, like __owner_client_id); a future algorithm change is
|
||||
* forward-safe — old rows just stop matching, which means "no coalesce".
|
||||
*/
|
||||
export function computeParamHash(data: Record<string, unknown>): string {
|
||||
const filtered: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(data)) {
|
||||
if (PARAM_HASH_EXCLUDED_KEYS.has(k)) continue;
|
||||
filtered[k] = data[k];
|
||||
}
|
||||
return createHash('sha256').update(stableStringify(filtered)).digest('hex');
|
||||
}
|
||||
|
||||
export interface AdmissionPolicy {
|
||||
/** Param-coalescing on for this name (parentless submits only). */
|
||||
coalesceParams: boolean;
|
||||
/** Waiting-TTL in hours; null = no TTL for this name. */
|
||||
ttlWaitingHours: number | null;
|
||||
/** Name-global max waiting; null = no quota for this name. */
|
||||
quotaMaxWaiting: number | null;
|
||||
}
|
||||
|
||||
export function admissionKilled(): boolean {
|
||||
return process.env.GBRAIN_MINIONS_ADMISSION === '0';
|
||||
}
|
||||
|
||||
function isOffValue(v: string): boolean {
|
||||
// Case-insensitive + trimmed: an operator typing 'FALSE' or 'Off' means
|
||||
// OFF — an emergency off-switch that only matches exact lowercase tokens
|
||||
// silently stays ON (structured-review finding).
|
||||
const t = v.trim().toLowerCase();
|
||||
return t === 'false' || t === '0' || t === 'off' || t === 'no';
|
||||
}
|
||||
|
||||
/**
|
||||
* Gate for embedding a job name into a COPY-PASTEABLE command hint
|
||||
* (`gbrain config set minions.…<name> …`). Display sanitization strips
|
||||
* control bytes but keeps shell metacharacters; a name like `x$(cmd)` must
|
||||
* never ride into a hint an operator will paste into a shell. Returns the
|
||||
* name when it is a safe config-key segment, else null (caller renders a
|
||||
* placeholder).
|
||||
*/
|
||||
export function safeConfigSegment(name: string): string | null {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(name) ? name : null;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(v: string | null): number | null {
|
||||
if (v == null || v.trim() === '') return null;
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n) || n <= 0) return null; // 0/garbage = disabled
|
||||
return n;
|
||||
}
|
||||
|
||||
// ~60s in-process cache: add() runs per submission; a config read per submit
|
||||
// would add a query to the hot path for a value that changes at human speed.
|
||||
type CacheEntry = { at: number; policy: AdmissionPolicy };
|
||||
const policyCache = new Map<string, CacheEntry>();
|
||||
const POLICY_CACHE_MS = 60_000;
|
||||
let warnedFailOpen = false;
|
||||
|
||||
/** Test seam: drop the cache so config changes are visible immediately. */
|
||||
export function _resetAdmissionCacheForTest(): void {
|
||||
policyCache.clear();
|
||||
warnedFailOpen = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the admission policy for a job name: config overrides > per-name
|
||||
* defaults tables. Fail-open to defaults on any config error (warn once per
|
||||
* process). Kill-switch returns the all-off policy.
|
||||
*/
|
||||
export async function resolveAdmissionPolicy(engine: BrainEngine, jobName: string): Promise<AdmissionPolicy> {
|
||||
if (admissionKilled()) {
|
||||
return { coalesceParams: false, ttlWaitingHours: null, quotaMaxWaiting: null };
|
||||
}
|
||||
const cached = policyCache.get(jobName);
|
||||
if (cached && Date.now() - cached.at < POLICY_CACHE_MS) return cached.policy;
|
||||
|
||||
const policy: AdmissionPolicy = {
|
||||
coalesceParams: PARAM_COALESCE_DEFAULT[jobName] === true,
|
||||
ttlWaitingHours: WAITING_TTL_DEFAULT_HOURS[jobName] ?? null,
|
||||
quotaMaxWaiting: QUOTA_MAX_WAITING_DEFAULT[jobName] ?? null,
|
||||
};
|
||||
try {
|
||||
const [coalesceV, ttlV, quotaV] = await Promise.all([
|
||||
engine.getConfig(`minions.coalesce_params.${jobName}`),
|
||||
engine.getConfig(`minions.ttl_waiting_hours.${jobName}`),
|
||||
engine.getConfig(`minions.quota_max_waiting.${jobName}`),
|
||||
]);
|
||||
if (coalesceV != null && coalesceV.trim() !== '') {
|
||||
policy.coalesceParams = !isOffValue(coalesceV.trim());
|
||||
}
|
||||
if (ttlV != null && ttlV.trim() !== '') {
|
||||
policy.ttlWaitingHours = parsePositiveNumber(ttlV);
|
||||
}
|
||||
if (quotaV != null && quotaV.trim() !== '') {
|
||||
policy.quotaMaxWaiting = parsePositiveNumber(quotaV);
|
||||
}
|
||||
} catch (e) {
|
||||
if (!warnedFailOpen) {
|
||||
warnedFailOpen = true;
|
||||
console.error(
|
||||
`[minions admission] config read failed (${e instanceof Error ? e.message : String(e)}) — ` +
|
||||
`using built-in defaults (coalesce=${policy.coalesceParams}, ttl=${policy.ttlWaitingHours ?? 'off'}h, ` +
|
||||
`quota=${policy.quotaMaxWaiting ?? 'off'}). Warning prints once per process.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
policyCache.set(jobName, { at: Date.now(), policy });
|
||||
return policy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Names with an active waiting-TTL: defaults ∪ config overrides discovered
|
||||
* via listConfigKeys('minions.ttl_waiting_hours.') when the engine supports
|
||||
* it (optional method — same guard pattern as config.ts's key listing).
|
||||
* Returns name → hours, with 0/garbage-configured names removed.
|
||||
*/
|
||||
export async function resolveTtlNames(engine: BrainEngine): Promise<Map<string, number>> {
|
||||
if (admissionKilled()) return new Map();
|
||||
const out = new Map<string, number>();
|
||||
for (const [name, hours] of Object.entries(WAITING_TTL_DEFAULT_HOURS)) out.set(name, hours);
|
||||
try {
|
||||
const listConfigKeys = (engine as { listConfigKeys?: (prefix: string) => Promise<string[]> }).listConfigKeys;
|
||||
if (typeof listConfigKeys === 'function') {
|
||||
const prefix = 'minions.ttl_waiting_hours.';
|
||||
const keys = await listConfigKeys.call(engine, prefix);
|
||||
for (const key of keys) {
|
||||
const name = key.slice(prefix.length);
|
||||
if (!name) continue;
|
||||
const v = parsePositiveNumber(await engine.getConfig(key));
|
||||
if (v == null) out.delete(name); // configured 0/garbage = disabled
|
||||
else out.set(name, v);
|
||||
}
|
||||
} else {
|
||||
// No listing support: still honor overrides for the DEFAULT names.
|
||||
for (const name of Object.keys(WAITING_TTL_DEFAULT_HOURS)) {
|
||||
const v = await engine.getConfig(`minions.ttl_waiting_hours.${name}`);
|
||||
if (v != null && v.trim() !== '') {
|
||||
const n = parsePositiveNumber(v);
|
||||
if (n == null) out.delete(name);
|
||||
else out.set(name, n);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fail open to the defaults already in `out`.
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count currently-waiting jobs already past their per-name TTL. Shared by
|
||||
* the worker's warn-before-act notice and the runPostUpgrade banner so the
|
||||
* two channels can never disagree on what "affected" means.
|
||||
*/
|
||||
export async function countTtlExpiredWaiting(
|
||||
engine: BrainEngine,
|
||||
ttlNames: Map<string, number>,
|
||||
): Promise<{ total: number; by_name: Record<string, number> }> {
|
||||
const by_name: Record<string, number> = {};
|
||||
let total = 0;
|
||||
for (const [name, hours] of ttlNames) {
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs
|
||||
WHERE name = $1 AND status = 'waiting' AND updated_at < now() - ($2 * interval '1 hour')`,
|
||||
[name, hours],
|
||||
);
|
||||
const n = parseInt(rows[0]?.count ?? '0', 10);
|
||||
by_name[name] = n;
|
||||
total += n;
|
||||
}
|
||||
return { total, by_name };
|
||||
}
|
||||
|
||||
/** Minimal structural dep so this module never imports MinionQueue (which imports us). */
|
||||
export interface WaitingTtlSweeper {
|
||||
handleWaitingTTL(opts?: { maxPerTick?: number }): Promise<{ cancelled: number; by_name: Record<string, number> }>;
|
||||
}
|
||||
|
||||
export interface WaitingTtlTickResult {
|
||||
/**
|
||||
* killed — GBRAIN_MINIONS_ADMISSION=0, nothing done.
|
||||
* notice — first tick: counted affected jobs, persisted the notice
|
||||
* timestamp; caller prints the warning. NOTHING cancelled.
|
||||
* grace — notice shown but the grace window hasn't elapsed; no sweep.
|
||||
* swept — sweep ran (cancelled may be 0).
|
||||
*/
|
||||
phase: 'killed' | 'notice' | 'grace' | 'swept';
|
||||
cancelled: number;
|
||||
by_name: Record<string, number>;
|
||||
/** Populated on phase 'notice': waiting jobs already past their TTL. */
|
||||
affected?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One warn-before-act TTL maintenance tick (user requirement D1A), extracted
|
||||
* from the worker interval so it is directly testable (pattern:
|
||||
* lock-renewal-tick.ts). Semantics:
|
||||
*
|
||||
* flag unset → count + stamp TTL_NOTICE_SHOWN_KEY with an ISO
|
||||
* timestamp, return 'notice' (caller warns; no sweep)
|
||||
* flag = ISO timestamp → sweep only after ttlNoticeGraceMs() has elapsed
|
||||
* since the notice ('grace' until then)
|
||||
* flag = 'true' (legacy / pre-grace releases) → sweep immediately
|
||||
*
|
||||
* The upgrade banner stamps the SAME key with the same ISO form, so an
|
||||
* interactive `gbrain upgrade` starts the clock too — whichever channel
|
||||
* shows the notice first, the operator gets a full grace window to set
|
||||
* `minions.ttl_waiting_hours.<name> 0` before anything is cancelled.
|
||||
*/
|
||||
export async function runWaitingTtlTick(
|
||||
engine: BrainEngine,
|
||||
sweeper: WaitingTtlSweeper,
|
||||
): Promise<WaitingTtlTickResult> {
|
||||
if (admissionKilled()) return { phase: 'killed', cancelled: 0, by_name: {} };
|
||||
const flag = (await engine.getConfig(TTL_NOTICE_SHOWN_KEY))?.trim() ?? '';
|
||||
if (flag === '') {
|
||||
const ttlNames = await resolveTtlNames(engine);
|
||||
const { total } = await countTtlExpiredWaiting(engine, ttlNames);
|
||||
await engine.setConfig(TTL_NOTICE_SHOWN_KEY, new Date().toISOString());
|
||||
return { phase: 'notice', cancelled: 0, by_name: {}, affected: total };
|
||||
}
|
||||
if (flag !== 'true') {
|
||||
const ts = Date.parse(flag);
|
||||
if (Number.isFinite(ts) && Date.now() - ts < ttlNoticeGraceMs()) {
|
||||
return { phase: 'grace', cancelled: 0, by_name: {} };
|
||||
}
|
||||
}
|
||||
const res = await sweeper.handleWaitingTTL();
|
||||
return { phase: 'swept', ...res };
|
||||
}
|
||||
+294
-14
@@ -16,6 +16,14 @@ import type {
|
||||
import { rowToMinionJob, rowToInboxMessage, rowToAttachment } from './types.ts';
|
||||
import { validateAttachment } from './attachments.ts';
|
||||
import { isProtectedJobName } from './protected-names.ts';
|
||||
import {
|
||||
computeParamHash,
|
||||
resolveAdmissionPolicy,
|
||||
resolveTtlNames,
|
||||
QueueQuotaExceededError,
|
||||
PARAM_HASH_EXCLUDED_KEYS,
|
||||
TTL_REASON_PREFIX,
|
||||
} from './admission.ts';
|
||||
import {
|
||||
defaultTimeoutMsFor, HANDLER_DEFAULT_TIMEOUT_MS,
|
||||
defaultLockDurationMsFor, HANDLER_DEFAULT_LOCK_DURATION_MS, clampLockDurationMs,
|
||||
@@ -106,6 +114,8 @@ type CoalesceAuditEvent = {
|
||||
queue: string; name: string; returned_job_id: number;
|
||||
waiting_count?: number; max_waiting?: number;
|
||||
pending_count?: number; max_pending?: number;
|
||||
/** Set when the coalesce matched on an identical payload hash (admission). */
|
||||
param_hash?: string;
|
||||
};
|
||||
|
||||
/** Shared cap-hit coalesce return for the backpressure guards: hydrate the
|
||||
@@ -219,6 +229,53 @@ export class MinionQueue {
|
||||
const delayUntil = opts?.delay ? new Date(Date.now() + opts.delay) : null;
|
||||
const maxSpawnDepth = opts?.max_spawn_depth ?? this.maxSpawnDepth;
|
||||
|
||||
// Admission policy (param-coalescing + name-global quota). Resolved
|
||||
// OUTSIDE the transaction (60s in-process cache; fail-open to defaults).
|
||||
// Parented submits never coalesce: fanout children belong to their
|
||||
// parent's bookkeeping/aggregator — returning some other child would
|
||||
// corrupt child_done accounting. opts.coalesce_params overrides per call.
|
||||
const policy = await resolveAdmissionPolicy(this.engine, jobName);
|
||||
// An EMPTY payload (after excluding the hash key itself) carries no
|
||||
// dedupe signal — two no-param submits are more likely distinct
|
||||
// placeholder/scaffolding jobs than a runaway producer (which always
|
||||
// carries a prompt). Never coalesce those.
|
||||
// A caller-supplied idempotency_key also disables param-coalescing:
|
||||
// producer-owned idempotency is the STRONGER contract ("this exact key
|
||||
// maps to this exact row"), and a param-coalesce hit would return a row
|
||||
// the key was never registered against — a later same-key submit would
|
||||
// then insert fresh and run the work twice (adversarial-review finding).
|
||||
const hashablePayload = Object.keys(data ?? {}).some(k => !PARAM_HASH_EXCLUDED_KEYS.has(k));
|
||||
const coalesceActive =
|
||||
(opts?.coalesce_params ?? policy.coalesceParams) &&
|
||||
!opts?.parent_job_id &&
|
||||
!opts?.idempotency_key &&
|
||||
hashablePayload &&
|
||||
childStatus === 'waiting';
|
||||
let paramHash: string | null = null;
|
||||
if (coalesceActive) {
|
||||
// Execution options are part of the coalescing IDENTITY (codex re-review
|
||||
// P1): identical payloads with different timeout/priority/attempt/
|
||||
// quiet-hours semantics are NOT the same job — coalescing them would
|
||||
// silently hand the second submitter the first's execution contract.
|
||||
// Only DEFINED options fold in (conservative: an explicit value never
|
||||
// coalesces onto an implicit-default row; forward-safe like any hash
|
||||
// input change — old rows just stop matching).
|
||||
const optIdentity: Record<string, unknown> = {};
|
||||
for (const k of ['priority', 'timeout_ms', 'max_attempts', 'quiet_hours', 'lock_duration_ms', 'delay_ms', 'max_stalled'] as const) {
|
||||
const v = (opts as Record<string, unknown> | undefined)?.[k];
|
||||
if (v !== undefined) optIdentity[k] = v;
|
||||
}
|
||||
paramHash = computeParamHash(
|
||||
Object.keys(optIdentity).length > 0
|
||||
? { ...(data ?? {}), __opts_identity: optIdentity }
|
||||
: ((data ?? {}) as Record<string, unknown>),
|
||||
);
|
||||
// Clone rather than mutate the caller's object; the hash rides in the
|
||||
// payload (the __-prefixed embedded-metadata convention) so the SQL
|
||||
// match needs no DDL and `jobs get` shows what matched.
|
||||
data = { ...(data ?? {}), __param_hash: paramHash };
|
||||
}
|
||||
|
||||
// Set inside the transaction by a cap-hit coalesce; flushed AFTER commit
|
||||
// so audit filesystem I/O never runs while holding the advisory lock.
|
||||
let coalesceAudit: CoalesceAuditEvent | null = null;
|
||||
@@ -251,6 +308,75 @@ export class MinionQueue {
|
||||
}
|
||||
}
|
||||
|
||||
// 1a. Param-coalescing (admission): an identical parentless submit —
|
||||
// same (name, queue, payload hash, incl. __owner_client_id so owner
|
||||
// lanes never cross) — returns the newest matching WAITING row instead
|
||||
// of inserting a duplicate. Honest-dispatch contract holds (coalesced:
|
||||
// true), and unlike the cap-hit coalesce below, returning this row to a
|
||||
// result-consumer is semantically exact: identical params ⇒ identical
|
||||
// result. Waiting-only by design (a RUNNING identical job does not
|
||||
// suppress a re-run; maxPending exists for single-flight callers).
|
||||
// Age-bounded to ttl/2: coalescing onto a nearly-TTL-expired row would
|
||||
// silently kill the fresh intent an hour later (round-2 V7).
|
||||
if (coalesceActive && paramHash) {
|
||||
const admissionQueue = opts?.queue ?? 'default';
|
||||
await tx.executeRaw(
|
||||
`SELECT pg_advisory_xact_lock(hashtext('minion_admission:' || $1 || ':' || $2 || ':' || $3))`,
|
||||
[jobName, admissionQueue, paramHash]
|
||||
);
|
||||
const ttlHours = policy.ttlWaitingHours;
|
||||
// updated_at, matching the TTL sweep's key: a requeued row has a
|
||||
// fresh TTL window and is a legitimate coalesce target again.
|
||||
const ageCond = ttlHours != null
|
||||
? `AND updated_at > now() - ($4 * interval '1 hour')`
|
||||
: '';
|
||||
const matchParams: unknown[] = [jobName, admissionQueue, paramHash];
|
||||
if (ttlHours != null) matchParams.push(ttlHours / 2);
|
||||
const match = await tx.executeRaw<Record<string, unknown>>(
|
||||
`SELECT * FROM minion_jobs
|
||||
WHERE name = $1 AND queue = $2 AND status = 'waiting'
|
||||
AND parent_job_id IS NULL
|
||||
AND data->>'__param_hash' = $3
|
||||
${ageCond}
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1`,
|
||||
matchParams
|
||||
);
|
||||
if (match.length > 0) {
|
||||
return coalesceReturn(match[0], {
|
||||
queue: admissionQueue,
|
||||
name: jobName,
|
||||
param_hash: paramHash,
|
||||
}, ev => { coalesceAudit = ev; });
|
||||
}
|
||||
}
|
||||
|
||||
// 1a2. Name-global waiting quota (admission; config-only, no shipped
|
||||
// default — user decision D2C). Counts the name across ALL queues:
|
||||
// fanout producers use per-run private queues (dream-inline-*), so a
|
||||
// queue-scoped count would reset per run and never bind. REJECTION,
|
||||
// not coalesce — quota-coalescing would hand result-consumers an
|
||||
// unrelated row. The name-global advisory lock below serializes
|
||||
// check+insert across concurrent submitters (adversarial finding:
|
||||
// without it, N parallel distinct-payload submits each observed
|
||||
// capacity and inserted, so overshoot was bounded only by attacker
|
||||
// concurrency — defeating the DoS backstop). Serialization cost only
|
||||
// applies to names with a quota configured, i.e. the runaway ones.
|
||||
if (policy.quotaMaxWaiting != null) {
|
||||
await tx.executeRaw(
|
||||
`SELECT pg_advisory_xact_lock(hashtext('minion_quota:' || $1))`,
|
||||
[jobName]
|
||||
);
|
||||
const quotaRows = await tx.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs WHERE name = $1 AND status = 'waiting'`,
|
||||
[jobName]
|
||||
);
|
||||
const waitingTotal = parseInt(quotaRows[0]?.count ?? '0', 10);
|
||||
if (waitingTotal >= policy.quotaMaxWaiting) {
|
||||
throw new QueueQuotaExceededError(jobName, waitingTotal, policy.quotaMaxWaiting);
|
||||
}
|
||||
}
|
||||
|
||||
// 1b. Submission-time backpressure for high-frequency named jobs.
|
||||
// Two guards share the advisory-lock machinery but differ in what they
|
||||
// count and how they scope:
|
||||
@@ -578,10 +704,40 @@ export class MinionQueue {
|
||||
* Returns the *root* (the job matching id), not an arbitrary descendant.
|
||||
*/
|
||||
async cancelJob(id: number): Promise<MinionJob | null> {
|
||||
const cancelled = await this.cancelJobs([id]);
|
||||
const root = cancelled.find(j => j.id === id);
|
||||
return root ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch variant of cancelJob: cancels every root id AND its descendants in
|
||||
* ONE transaction, with the full bookkeeping the single-id path carries
|
||||
* (child_done inbox messages + aggregator-parent resolution). Callers that
|
||||
* cancel in bulk (the waiting-TTL sweep) MUST use this — a raw set-based
|
||||
* UPDATE would skip that bookkeeping and wedge parents in waiting-children
|
||||
* forever (the exact wedge class this wave fights).
|
||||
*
|
||||
* `opts.reason` is written to error_text (COALESCE-preserved when a row
|
||||
* already carries one). The single-id UPDATE never wrote error_text, so
|
||||
* every surface keyed on a reason prefix (jobs stats, doctor) would
|
||||
* silently report zero without this parameter.
|
||||
*/
|
||||
async cancelJobs(ids: number[], opts?: { reason?: string; rootStatuses?: MinionJobStatus[] }): Promise<MinionJob[]> {
|
||||
if (ids.length === 0) return [];
|
||||
// opts.rootStatuses re-checks each ROOT id's status ATOMICALLY inside the
|
||||
// cancel UPDATE's CTE seed. The waiting-TTL sweep passes ['waiting'] to
|
||||
// close its SELECT→cancel race: claim() and the sweep both target the
|
||||
// oldest waiting rows, so without this a job claimed between the sweep's
|
||||
// SELECT and this UPDATE would be cancelled while ACTIVE (lock_token
|
||||
// NULLed under the running handler). Operator cancels omit it — killing
|
||||
// an active job is exactly what `jobs cancel` means.
|
||||
const rootStatuses = opts?.rootStatuses ?? null;
|
||||
return this.engine.transaction(async (tx) => {
|
||||
const rows = await tx.executeRaw<Record<string, unknown>>(
|
||||
`WITH RECURSIVE descendants AS (
|
||||
SELECT id, 0 AS d FROM minion_jobs WHERE id = $1
|
||||
SELECT id, 0 AS d FROM minion_jobs
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ($3::text[] IS NULL OR status = ANY($3::text[]))
|
||||
UNION ALL
|
||||
SELECT m.id, descendants.d + 1
|
||||
FROM minion_jobs m
|
||||
@@ -592,14 +748,20 @@ export class MinionQueue {
|
||||
status = 'cancelled',
|
||||
lock_token = NULL,
|
||||
lock_until = NULL,
|
||||
-- Reason stamps ROOT ids only: a descendant is bookkeeping-cancelled
|
||||
-- because its parent went away, not because IT hit the caller's
|
||||
-- reason (e.g. a waiting-TTL child would otherwise carry a factually
|
||||
-- false 'waited > Nh' text AND inflate the LIKE-prefix stats the
|
||||
-- alerting surfaces count).
|
||||
error_text = CASE WHEN id = ANY($1::int[]) THEN COALESCE($2, error_text) ELSE error_text END,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id IN (SELECT id FROM descendants)
|
||||
AND status IN ('waiting','active','delayed','waiting-children','paused')
|
||||
RETURNING *`,
|
||||
[id]
|
||||
[ids, opts?.reason ?? null, rootStatuses]
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
// v0.15: emit child_done(outcome='cancelled') for every cancelled row
|
||||
// that had a parent. Without this, an aggregator waiting for N
|
||||
@@ -652,11 +814,68 @@ export class MinionQueue {
|
||||
);
|
||||
}
|
||||
|
||||
const root = rows.find(r => (r.id as number) === id);
|
||||
return root ? rowToMinionJob(root) : null;
|
||||
return rows.map(rowToMinionJob);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Waiting-TTL sweep (admission control, run from the worker's maintenance
|
||||
* interval): cancel jobs still WAITING past their per-name TTL, via
|
||||
* cancelJobs() so descendants cancel and aggregator parents resolve.
|
||||
*
|
||||
* `maxPerTick` bounds one tick's work (default 500) so the first
|
||||
* post-upgrade tick against a large backlog can't stall the maintenance
|
||||
* loop — the backlog drains over a few ticks. Oldest-first so FIFO
|
||||
* fairness of what REMAINS is preserved.
|
||||
*
|
||||
* NOTE (warn-before-act, user requirement D1A): the worker gates this
|
||||
* sweep behind the one-time `minions.ttl_notice_shown` flag — tick 1
|
||||
* counts + warns, sweeping starts on tick 2. The gate lives in worker.ts
|
||||
* (the channel where the notice prints); this method just sweeps.
|
||||
*/
|
||||
async handleWaitingTTL(opts?: { maxPerTick?: number }): Promise<{ cancelled: number; by_name: Record<string, number> }> {
|
||||
const maxPerTick = Math.max(1, Math.floor(opts?.maxPerTick ?? 500));
|
||||
const ttlNames = await resolveTtlNames(this.engine);
|
||||
const by_name: Record<string, number> = {};
|
||||
let cancelled = 0;
|
||||
for (const [name, hours] of ttlNames) {
|
||||
if (cancelled >= maxPerTick) break;
|
||||
const budget = maxPerTick - cancelled;
|
||||
// Keyed on updated_at (last state transition), NOT created_at: every
|
||||
// path that RETURNS a row to 'waiting' — handleStalled's requeue (sweep
|
||||
// #1 of the SAME maintenance tick), retryJob, promoteDelayed, the
|
||||
// parent-unblock flips — bumps updated_at but not created_at. Keying on
|
||||
// created_at would cancel a job the same tick just requeued it, with a
|
||||
// "waited > Nh" reason that is factually false. For rows that sat
|
||||
// untouched in 'waiting' the two timestamps are equivalent, so the
|
||||
// backlog-drain semantics are unchanged.
|
||||
const stale = await this.engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM minion_jobs
|
||||
WHERE name = $1 AND status = 'waiting'
|
||||
AND updated_at < now() - ($2 * interval '1 hour')
|
||||
ORDER BY updated_at ASC
|
||||
LIMIT $3`,
|
||||
[name, hours, budget]
|
||||
);
|
||||
if (stale.length === 0) continue;
|
||||
const reason =
|
||||
`${TTL_REASON_PREFIX}: waited > ${hours}h in queue ` +
|
||||
`(minions.ttl_waiting_hours.${name}; set 0 to disable)`;
|
||||
// rootStatuses:['waiting'] re-checks atomically inside the cancel — a
|
||||
// job CLAIMED between the SELECT above and this UPDATE must not be
|
||||
// cancelled mid-run (both the claimer and this sweep target the oldest
|
||||
// waiting rows, so the race is systematic, not incidental).
|
||||
const swept = await this.cancelJobs(stale.map(r => r.id), { reason, rootStatuses: ['waiting'] });
|
||||
// Count only the requested roots — descendants of a swept parent are
|
||||
// bookkeeping, not TTL victims of their own.
|
||||
const rootIds = new Set(stale.map(r => r.id));
|
||||
const rootCount = swept.filter(j => rootIds.has(j.id)).length;
|
||||
by_name[name] = (by_name[name] ?? 0) + rootCount;
|
||||
cancelled += rootCount;
|
||||
}
|
||||
return { cancelled, by_name };
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-queue a failed or dead job for retry.
|
||||
*
|
||||
@@ -723,7 +942,19 @@ export class MinionQueue {
|
||||
/** Get job statistics. */
|
||||
async getStats(opts?: { since?: Date; queue?: string }): Promise<{
|
||||
by_status: Record<string, number>;
|
||||
by_type: Array<{ name: string; total: number; completed: number; failed: number; dead: number; avg_duration_ms: number | null }>;
|
||||
/**
|
||||
* Per-type window stats. `total` counts rows CREATED in the window
|
||||
* (intake). The `drained_*` fields count rows that reached a terminal
|
||||
* status IN the window (`finished_at >= since`) regardless of when they
|
||||
* were created — the true outflow. They are split by terminal status
|
||||
* because a naive combined "drain" number self-inflates on TTL/manual
|
||||
* cancellations while zero useful work happens; divergence alerting
|
||||
* compares intake against drained_completed. `waiting_now` and
|
||||
* `oldest_waiting_minutes` are point-in-time (not windowed).
|
||||
*/
|
||||
by_type: Array<{ name: string; total: number; completed: number; failed: number; dead: number; avg_duration_ms: number | null;
|
||||
drained_completed: number; drained_failed: number; drained_dead: number; drained_cancelled: number;
|
||||
waiting_now: number; oldest_waiting_minutes: number | null }>;
|
||||
queue_health: { waiting: number; active: number; stalled: number };
|
||||
/**
|
||||
* issue #1801 — QUEUE-SCOPED wedge signature for the `jobs stats` WEDGED
|
||||
@@ -763,14 +994,63 @@ export class MinionQueue {
|
||||
GROUP BY name ORDER BY total DESC`,
|
||||
[since.toISOString()]
|
||||
);
|
||||
const by_type = typeRows.map(r => ({
|
||||
name: r.name as string,
|
||||
total: parseInt(r.total as string, 10),
|
||||
completed: parseInt(r.completed as string, 10),
|
||||
failed: parseInt(r.failed as string, 10),
|
||||
dead: parseInt(r.dead as string, 10),
|
||||
avg_duration_ms: r.avg_duration_ms != null ? Math.round(r.avg_duration_ms as number) : null,
|
||||
}));
|
||||
// True per-type outflow: rows that reached a terminal status IN the
|
||||
// window, keyed on finished_at (a row created last week and finished
|
||||
// today drained today). Split by status — cancellations (incl. the
|
||||
// waiting-TTL sweep) are outflow but not useful work.
|
||||
const drainRows = await this.engine.executeRaw<Record<string, unknown>>(
|
||||
`SELECT name,
|
||||
count(*) FILTER (WHERE status = 'completed')::text AS d_completed,
|
||||
count(*) FILTER (WHERE status = 'failed')::text AS d_failed,
|
||||
count(*) FILTER (WHERE status = 'dead')::text AS d_dead,
|
||||
count(*) FILTER (WHERE status = 'cancelled')::text AS d_cancelled
|
||||
FROM minion_jobs
|
||||
WHERE finished_at IS NOT NULL AND finished_at >= $1
|
||||
AND status IN ('completed','failed','dead','cancelled')
|
||||
GROUP BY name`,
|
||||
[since.toISOString()]
|
||||
);
|
||||
const drainByName = new Map(drainRows.map(r => [r.name as string, r]));
|
||||
|
||||
// Point-in-time per-type waiting depth + oldest wait age.
|
||||
const depthRows = await this.engine.executeRaw<Record<string, unknown>>(
|
||||
`SELECT name,
|
||||
count(*)::text AS waiting_now,
|
||||
EXTRACT(EPOCH FROM (now() - min(created_at)))::text AS oldest_waiting_seconds
|
||||
FROM minion_jobs WHERE status = 'waiting'
|
||||
GROUP BY name`,
|
||||
[]
|
||||
);
|
||||
const depthByName = new Map(depthRows.map(r => [r.name as string, r]));
|
||||
|
||||
// Union of names so a type with waiting rows but zero window intake (or
|
||||
// vice versa) still gets a row — divergence alerting needs both sides.
|
||||
const typeNames = new Set<string>([
|
||||
...typeRows.map(r => r.name as string),
|
||||
...drainRows.map(r => r.name as string),
|
||||
...depthRows.map(r => r.name as string),
|
||||
]);
|
||||
const typeByName = new Map(typeRows.map(r => [r.name as string, r]));
|
||||
const by_type = [...typeNames].map(name => {
|
||||
const r = typeByName.get(name);
|
||||
const d = drainByName.get(name);
|
||||
const w = depthByName.get(name);
|
||||
const oldest = w?.oldest_waiting_seconds != null ? Number(w.oldest_waiting_seconds) : null;
|
||||
return {
|
||||
name,
|
||||
total: r ? parseInt(r.total as string, 10) : 0,
|
||||
completed: r ? parseInt(r.completed as string, 10) : 0,
|
||||
failed: r ? parseInt(r.failed as string, 10) : 0,
|
||||
dead: r ? parseInt(r.dead as string, 10) : 0,
|
||||
avg_duration_ms: r?.avg_duration_ms != null ? Math.round(r.avg_duration_ms as number) : null,
|
||||
drained_completed: d ? parseInt(d.d_completed as string, 10) : 0,
|
||||
drained_failed: d ? parseInt(d.d_failed as string, 10) : 0,
|
||||
drained_dead: d ? parseInt(d.d_dead as string, 10) : 0,
|
||||
drained_cancelled: d ? parseInt(d.d_cancelled as string, 10) : 0,
|
||||
waiting_now: w ? parseInt(w.waiting_now as string, 10) : 0,
|
||||
oldest_waiting_minutes: oldest != null && Number.isFinite(oldest) ? Math.round(oldest / 60) : null,
|
||||
};
|
||||
}).sort((a, b) => b.total - a.total);
|
||||
|
||||
// Queue health: stalled = active with expired lock
|
||||
const stalledRows = await this.engine.executeRaw<{ count: string }>(
|
||||
|
||||
@@ -163,6 +163,17 @@ export interface MinionJobInput {
|
||||
* as a public submit flag yet — semantics exclude delayed/paused/
|
||||
* waiting-children rows deliberately. */
|
||||
maxPending?: number;
|
||||
/**
|
||||
* Admission param-coalescing override. When unset, the per-name default
|
||||
* (admission.ts PARAM_COALESCE_DEFAULT, config-overridable via
|
||||
* minions.coalesce_params.<name>) applies — on for 'subagent'. When
|
||||
* active, a parentless submit whose payload hash (sha256 of
|
||||
* stable-stringified data, __owner_client_id INCLUDED so owner lanes never
|
||||
* cross) matches a WAITING row for the same (name, queue) returns that row
|
||||
* with `coalesced: true` instead of inserting a duplicate. Parented
|
||||
* submits never coalesce regardless of this flag.
|
||||
*/
|
||||
coalesce_params?: boolean;
|
||||
|
||||
// v12: scheduler polish
|
||||
/**
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
ABORT_REASON_LOCK_LOST,
|
||||
} from './types.ts';
|
||||
import { MinionQueue } from './queue.ts';
|
||||
import { runWaitingTtlTick, ttlNoticeGraceMs } from './admission.ts';
|
||||
import { calculateBackoff } from './backoff.ts';
|
||||
import { RateLeaseUnavailableError } from './handlers/subagent.ts';
|
||||
import { logLeasePressure } from './lease-pressure-audit.ts';
|
||||
@@ -438,6 +439,33 @@ export class MinionWorker extends EventEmitter {
|
||||
console.error('Wall-clock timeout detection error:', e instanceof Error ? e.message : String(e));
|
||||
await recoverConnection('handleWallClockTimeouts', e);
|
||||
}
|
||||
// 4th sweep: waiting-TTL (admission control). Warn-before-act (user
|
||||
// requirement D1A) lives in runWaitingTtlTick (admission.ts): the first
|
||||
// tick counts + stamps the notice timestamp, sweeping starts only after
|
||||
// the grace window elapses. The gate is engine-state (not process
|
||||
// state) because the worker restarts via self-upgrade/systemd without
|
||||
// ever running the CLI's runPostUpgrade banner; the notice must precede
|
||||
// the cancellation on EVERY channel, and the worker log is the daemon
|
||||
// channel.
|
||||
try {
|
||||
const tick = await runWaitingTtlTick(this.engine, this.queue);
|
||||
if (tick.phase === 'notice') {
|
||||
console.log(
|
||||
`⚠ Waiting-TTL is now active: ${tick.affected ?? 0} queued job(s) currently exceed their TTL and ` +
|
||||
`will be cancelled after a ${Math.round(ttlNoticeGraceMs() / 60_000)}min grace window. ` +
|
||||
`Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>.`,
|
||||
);
|
||||
} else if (tick.phase === 'swept' && tick.cancelled > 0) {
|
||||
const breakdown = Object.entries(tick.by_name).map(([n, c]) => `${n}: ${c}`).join(', ');
|
||||
console.log(
|
||||
`Waiting-TTL: cancelled ${tick.cancelled} job(s) that waited past their TTL (${breakdown}). ` +
|
||||
`Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>. See 'gbrain jobs stats'.`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Waiting-TTL sweep error:', e instanceof Error ? e.message : String(e));
|
||||
await recoverConnection('handleWaitingTTL', e);
|
||||
}
|
||||
}, this.opts.stalledInterval);
|
||||
|
||||
// Periodic RSS watchdog — closes the production-freeze regression where
|
||||
|
||||
@@ -181,6 +181,9 @@ const extraction_review: Operation = {
|
||||
if (ctx.dryRun) return { dry_run: true, action: `extraction_review:${action}`, slugs };
|
||||
const results: Array<{ slug: string; status: string }> = [];
|
||||
for (const slug of slugs) {
|
||||
// First-match read is safe here: BOTH writes below key on the RETURNED
|
||||
// row's page.source_id, so read and write can never target different
|
||||
// rows. gbrain-allow-unscoped-getpage: write follows the returned row
|
||||
const page = await ctx.engine.getPage(slug, ctx.sourceId ? { sourceId: ctx.sourceId } : undefined);
|
||||
if (!page) {
|
||||
results.push({ slug, status: 'not_found' });
|
||||
|
||||
+24
-6
@@ -328,12 +328,24 @@ const submit_agent: Operation = {
|
||||
);
|
||||
}
|
||||
if (delegatedSource) jobData.source_id = delegatedSource;
|
||||
const job = await queue.add(
|
||||
'subagent',
|
||||
jobData,
|
||||
{ queue: (p.queue as string) || 'default' },
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
let job;
|
||||
try {
|
||||
job = await queue.add(
|
||||
'subagent',
|
||||
jobData,
|
||||
{ queue: (p.queue as string) || 'default' },
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
} catch (e) {
|
||||
// Admission quota (minions.quota_max_waiting.subagent, config-only):
|
||||
// surface as a structured retryable error, not an opaque internal one.
|
||||
// The quota message already omits live cross-tenant queue depth.
|
||||
const { isQueueQuotaExceededError } = await import('../minions/admission.ts');
|
||||
if (isQueueQuotaExceededError(e)) {
|
||||
throw new OperationError('rate_limited', e.message, 'Retry after the queue drains, or ask the operator to raise the quota.');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Audit trail (D4) — best-effort JSONL.
|
||||
try {
|
||||
@@ -360,6 +372,12 @@ const submit_agent: Operation = {
|
||||
id: job.id,
|
||||
name: 'subagent',
|
||||
client_id: clientId,
|
||||
// Honest-dispatch: true when this submit was param-coalesced onto an
|
||||
// existing WAITING job with identical params (same owner lane) instead
|
||||
// of enqueuing a new one. Clients wanting N independent runs of one
|
||||
// prompt should vary the params (adversarial-review finding — the flag
|
||||
// makes the suppression detectable rather than silent).
|
||||
...(job.coalesced === true ? { coalesced: true } : {}),
|
||||
queue_state: await probeQueueStateSafe(ctx, job.queue, ['subagent']),
|
||||
};
|
||||
},
|
||||
|
||||
@@ -78,7 +78,18 @@ export class SlugRegistryError extends Error {
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u');
|
||||
|
||||
export class SlugRegistry {
|
||||
constructor(private engine: BrainEngine) {}
|
||||
/**
|
||||
* `sourceId` scopes every existence probe to the SAME source the paired
|
||||
* putPage will write to (engine.putPage defaults to 'default' when unset).
|
||||
* Pre-fix the probes were UNSCOPED — getPage matched a slug in ANY source,
|
||||
* so a slug taken only in source B forced a spurious disambiguation (or a
|
||||
* false isFree=false) for a write that was going to land in source A.
|
||||
*/
|
||||
constructor(private engine: BrainEngine, private sourceId?: string) {}
|
||||
|
||||
private scope(): { sourceId: string } {
|
||||
return { sourceId: this.sourceId ?? 'default' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new slug, or disambiguate if taken. Checks engine.getPage(slug)
|
||||
@@ -93,7 +104,7 @@ export class SlugRegistry {
|
||||
}
|
||||
|
||||
// Fast path: desired is free
|
||||
const existing = await this.engine.getPage(desiredSlug);
|
||||
const existing = await this.engine.getPage(desiredSlug, this.scope());
|
||||
if (!existing) {
|
||||
return { slug: desiredSlug, exact: true };
|
||||
}
|
||||
@@ -110,7 +121,7 @@ export class SlugRegistry {
|
||||
// append-numeric disambiguation: start at 2 (matches "alice-smith" → "alice-smith-2")
|
||||
for (let n = 2; n <= maxDisambiguator; n++) {
|
||||
const candidate = `${desiredSlug}-${n}`;
|
||||
const conflict = await this.engine.getPage(candidate);
|
||||
const conflict = await this.engine.getPage(candidate, this.scope());
|
||||
if (!conflict) {
|
||||
return { slug: candidate, exact: false, disambiguator: n };
|
||||
}
|
||||
@@ -129,7 +140,7 @@ export class SlugRegistry {
|
||||
*/
|
||||
async isFree(slug: string): Promise<boolean> {
|
||||
if (!SLUG_RE.test(slug)) return false;
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
return !existing;
|
||||
}
|
||||
|
||||
|
||||
+35
-12
@@ -37,6 +37,15 @@ export interface BrainWriterOptions {
|
||||
* follow-on release after soak).
|
||||
*/
|
||||
strictMode?: StrictMode;
|
||||
/**
|
||||
* Source every read AND write in this writer targets. Default 'default'
|
||||
* (matches engine.putPage's schema default). Pre-fix the writer's reads
|
||||
* were UNSCOPED (first slug match across ANY source) while its writes
|
||||
* landed in 'default' — the exact unscoped-check/scoped-write bug class:
|
||||
* setCompiledTruth could read source B's page and clobber the default
|
||||
* source's row with it.
|
||||
*/
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface EntityInput {
|
||||
@@ -128,8 +137,14 @@ class WriteTxImpl implements WriteTx {
|
||||
constructor(
|
||||
private engine: BrainEngine,
|
||||
public readonly context: ResolverContext,
|
||||
private sourceId?: string,
|
||||
) {
|
||||
this.slugRegistry = new SlugRegistry(engine);
|
||||
this.slugRegistry = new SlugRegistry(engine, sourceId);
|
||||
}
|
||||
|
||||
/** Read+write scope: mirrors engine.putPage's implicit 'default'. */
|
||||
private scope(): { sourceId: string } {
|
||||
return { sourceId: this.sourceId ?? 'default' };
|
||||
}
|
||||
|
||||
async createEntity(input: EntityInput): Promise<string> {
|
||||
@@ -163,18 +178,18 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: input.compiledTruth,
|
||||
timeline: input.timeline ?? '',
|
||||
frontmatter: input.frontmatter ?? {},
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
return slug;
|
||||
}
|
||||
|
||||
async appendTimeline(slug: string, entry: TimelineInput): Promise<void> {
|
||||
await this.engine.addTimelineEntry(slug, entry); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface — output gets fenced into pages via putPage in the same transaction
|
||||
await this.engine.addTimelineEntry(slug, entry, this.scope()); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface — output gets fenced into pages via putPage in the same transaction
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async setCompiledTruth(slug: string, body: string): Promise<void> {
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
if (!existing) throw new WriteError('invalid_input', `setCompiledTruth: page not found: ${slug}`);
|
||||
await this.engine.putPage(slug, {
|
||||
type: existing.type,
|
||||
@@ -182,12 +197,12 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: body,
|
||||
timeline: existing.timeline,
|
||||
frontmatter: existing.frontmatter,
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async setFrontmatterField(slug: string, key: string, value: unknown): Promise<void> {
|
||||
const existing = await this.engine.getPage(slug);
|
||||
const existing = await this.engine.getPage(slug, this.scope());
|
||||
if (!existing) throw new WriteError('invalid_input', `setFrontmatterField: page not found: ${slug}`);
|
||||
const nextFm = { ...existing.frontmatter, [key]: value };
|
||||
await this.engine.putPage(slug, {
|
||||
@@ -196,21 +211,26 @@ class WriteTxImpl implements WriteTx {
|
||||
compiled_truth: existing.compiled_truth,
|
||||
timeline: existing.timeline,
|
||||
frontmatter: nextFm,
|
||||
});
|
||||
}, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async putRawData(slug: string, source: string, data: object): Promise<void> {
|
||||
await this.engine.putRawData(slug, source, data);
|
||||
await this.engine.putRawData(slug, source, data, this.scope());
|
||||
this.touchedSlugs.add(slug);
|
||||
}
|
||||
|
||||
async addLink(from: string, to: string, context?: string, linkType?: string): Promise<void> {
|
||||
await this.engine.addLink(from, to, context, linkType); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface
|
||||
// Both endpoints scoped to this writer's source — synthesize-phase links
|
||||
// are within-source by definition, and unscoped endpoints resolve against
|
||||
// 'default'-source rows (wrong page or missing) in a scoped writer.
|
||||
const sid = this.scope().sourceId;
|
||||
const linkScope = { fromSourceId: sid, toSourceId: sid };
|
||||
await this.engine.addLink(from, to, context, linkType, undefined, undefined, undefined, linkScope); // gbrain-allow-direct-insert: BrainWriter is the canonical synthesize-phase write surface
|
||||
// Reverse back-link — both directions inside the same outer transaction.
|
||||
// Uses 'backlink' label on the reverse if no linkType was specified so
|
||||
// the reverse is distinguishable from the forward semantic type.
|
||||
await this.engine.addLink(to, from, context, linkType ? `${linkType}_back` : 'backlink'); // gbrain-allow-direct-insert: BrainWriter synthesize-phase reverse back-link in the same transaction as the forward addLink above
|
||||
await this.engine.addLink(to, from, context, linkType ? `${linkType}_back` : 'backlink', undefined, undefined, undefined, linkScope); // gbrain-allow-direct-insert: BrainWriter synthesize-phase reverse back-link in the same transaction as the forward addLink above
|
||||
this.touchedSlugs.add(from);
|
||||
this.touchedSlugs.add(to);
|
||||
}
|
||||
@@ -223,12 +243,14 @@ class WriteTxImpl implements WriteTx {
|
||||
export class BrainWriter {
|
||||
private validators: PageValidator[] = [];
|
||||
private strictMode: StrictMode;
|
||||
private sourceId?: string;
|
||||
|
||||
constructor(
|
||||
private engine: BrainEngine,
|
||||
opts: BrainWriterOptions = {},
|
||||
) {
|
||||
this.strictMode = opts.strictMode ?? 'lint';
|
||||
this.sourceId = opts.sourceId;
|
||||
}
|
||||
|
||||
register(validator: PageValidator): void {
|
||||
@@ -247,14 +269,15 @@ export class BrainWriter {
|
||||
|
||||
let report: ValidationReport | null = null;
|
||||
|
||||
const strictSourceId = this.sourceId;
|
||||
const txResult = await this.engine.transaction(async (txEngine) => {
|
||||
const tx = new WriteTxImpl(txEngine, ctx);
|
||||
const tx = new WriteTxImpl(txEngine, ctx, strictSourceId);
|
||||
const result = await fn(tx);
|
||||
|
||||
// Validators run before the outer transaction commits.
|
||||
if (strict !== 'off') {
|
||||
report = await runValidators(txEngine, validators, tx.touchedSlugs, {
|
||||
sourceId: 'default',
|
||||
sourceId: strictSourceId ?? 'default',
|
||||
});
|
||||
// `ctx.logger.info` would be nice but keep validator behavior uniform
|
||||
// regardless of strict/lint mode. Caller inspects the report.
|
||||
|
||||
+15
-3
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Eng-review fold: reuses the v0.17 `~/.gbrain/cycle.lock` PID-liveness
|
||||
* pattern (src/core/cycle.ts:acquireFileLock) but scoped per page so two
|
||||
* parallel `gbrain takes add` calls + a `takes seed --refresh` running in
|
||||
* parallel `gbrain takes add` calls + a refresh-mode `takes seed` running in
|
||||
* autopilot can't race on the same `<slug>.md` file.
|
||||
*
|
||||
* Lock file path: `~/.gbrain/page-locks/<sha256-of-slug>.lock`. SHA-256
|
||||
@@ -87,13 +87,25 @@ function tryAcquireOnce(slug: string, lockPath: string): PageLockHandle | null {
|
||||
if (pidAlive && ageMs < LOCK_TTL_MS) {
|
||||
return null; // live holder
|
||||
}
|
||||
// Stale — fall through to overwrite.
|
||||
// Stale — remove it, then race for the exclusive create below. Two
|
||||
// reclaimers can both unlink, but only ONE wins the 'wx' open; the
|
||||
// pre-fix existsSync→writeFileSync sequence let both "acquire"
|
||||
// (adversarial finding — the loser silently lost its writes).
|
||||
try { unlinkSync(lockPath); } catch { /* already gone */ }
|
||||
} catch {
|
||||
// Any read/stat error → treat as stale.
|
||||
try { unlinkSync(lockPath); } catch { /* already gone */ }
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(lockPath, `${pid}\n${new Date().toISOString()}\n`);
|
||||
// Exclusive create: mutual exclusion comes from O_EXCL, not from the
|
||||
// (racy) existence check above. Losing the create race = lock not held.
|
||||
try {
|
||||
writeFileSync(lockPath, `${pid}\n${new Date().toISOString()}\n`, { flag: 'wx' });
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code === 'EEXIST') return null;
|
||||
throw e;
|
||||
}
|
||||
|
||||
return {
|
||||
slug,
|
||||
|
||||
@@ -1455,9 +1455,13 @@ export class PGLiteEngine implements BrainEngine {
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
|
||||
FROM pages WHERE ${where.join(' AND ')}
|
||||
ORDER BY (source_id = 'default') DESC, source_id ASC
|
||||
LIMIT 1`,
|
||||
params
|
||||
);
|
||||
// Deterministic multi-source tiebreak — default-source-first, then stable
|
||||
// alpha. Engine parity: postgres-engine.ts carries the identical clause.
|
||||
if (rows.length === 0) return null;
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
@@ -1227,8 +1227,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
contextual_retrieval_mode
|
||||
FROM pages
|
||||
WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
|
||||
ORDER BY (source_id = 'default') DESC, source_id ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
// Deterministic multi-source tiebreak: without an ORDER BY, LIMIT 1 on a
|
||||
// slug that exists in several sources returned an ARBITRARY row.
|
||||
// Default-source-first (then stable alpha) — plain alpha would prefer
|
||||
// e.g. 'archive' over 'default'. Engine parity: pglite-engine.ts carries
|
||||
// the identical clause.
|
||||
if (rows.length === 0) return null;
|
||||
return rowToPage(rows[0]);
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import type { SchemaPackManifest } from './manifest-v1.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { readRecentMutations } from './mutate-audit.ts';
|
||||
import { classifyStoredType, sanitizeTypeForDisplay, safeCliToken } from './type-usage.ts';
|
||||
|
||||
export type LintSeverity = 'error' | 'warning';
|
||||
|
||||
@@ -37,6 +38,13 @@ export interface LintOpts {
|
||||
engine?: BrainEngine;
|
||||
/** Limit scan window for audit-aware rules. Default 7 days. */
|
||||
daysBack?: number;
|
||||
/**
|
||||
* Scope DB-aware STORED-TYPE rules to one source. On a multi-source brain
|
||||
* each source may resolve its own pack — comparing another source's rows
|
||||
* against THIS manifest produces false alias/undeclared warnings (codex
|
||||
* re-review). Omitted = global scan (single-source brains, status quo).
|
||||
*/
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export type LintRule = (manifest: SchemaPackManifest, opts?: LintOpts) =>
|
||||
@@ -348,6 +356,70 @@ export const linkRegexCatastrophicBacktrack: LintRule = (manifest) => {
|
||||
return issues;
|
||||
};
|
||||
|
||||
/**
|
||||
* Data-plane corpus audit (alias-footgun incident class): pages whose STORED
|
||||
* type is an alias of a canonical pack type. Explicit frontmatter types are
|
||||
* stored literally and never re-normalized, so alias-typed pages silently
|
||||
* diverge from the canonical filing. Warning severity — the import-time
|
||||
* surface (schema.type_warnings) catches new writes; this catches the
|
||||
* existing corpus.
|
||||
*/
|
||||
export const storedTypeIsAlias: LintRule = async (manifest, opts) => {
|
||||
if (!opts?.engine) return [];
|
||||
const issues: LintIssue[] = [];
|
||||
const rows = await opts.engine.executeRaw<{ type: string; n: string }>(
|
||||
`SELECT type, count(*)::text AS n FROM pages
|
||||
WHERE deleted_at IS NULL AND ($1::text IS NULL OR source_id = $1)
|
||||
GROUP BY type`,
|
||||
[opts.sourceId ?? null],
|
||||
);
|
||||
for (const r of rows) {
|
||||
const cls = classifyStoredType(r.type, manifest);
|
||||
if (cls.kind === 'alias_of') {
|
||||
const t = sanitizeTypeForDisplay(r.type);
|
||||
issues.push({
|
||||
rule: 'stored_type_is_alias',
|
||||
severity: 'warning',
|
||||
message: `${r.n} page(s) store type '${t}', an alias of '${cls.canonical}'${cls.directory ? ` (files under ${cls.directory})` : ''} — agents may route it inconsistently`,
|
||||
pack: manifest.name,
|
||||
type: r.type,
|
||||
// Real retype surface is the PROTECTED unify-types job — there is no
|
||||
// `schema unify` subcommand (dead-command class; red-team catch).
|
||||
// Flags spelled dash-less: the flag-registry generator harvests bare
|
||||
// dash-tokens from strings one import level deep.
|
||||
hint: `retype via the unify-types job (gbrain jobs submit unify-types; supports dry-run) OR declare '${t}' as its own page type (gbrain schema add-type)`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
};
|
||||
|
||||
/** Data-plane sibling: pages whose stored type is entirely undeclared in the pack. */
|
||||
export const storedTypeUndeclared: LintRule = async (manifest, opts) => {
|
||||
if (!opts?.engine) return [];
|
||||
const issues: LintIssue[] = [];
|
||||
const rows = await opts.engine.executeRaw<{ type: string; n: string }>(
|
||||
`SELECT type, count(*)::text AS n FROM pages
|
||||
WHERE deleted_at IS NULL AND ($1::text IS NULL OR source_id = $1)
|
||||
GROUP BY type`,
|
||||
[opts.sourceId ?? null],
|
||||
);
|
||||
for (const r of rows) {
|
||||
if (classifyStoredType(r.type, manifest).kind === 'undeclared') {
|
||||
const t = sanitizeTypeForDisplay(r.type);
|
||||
issues.push({
|
||||
rule: 'stored_type_undeclared',
|
||||
severity: 'warning',
|
||||
message: `${r.n} page(s) store type '${t}' which is not declared in the active pack (not a page_type, not an alias)`,
|
||||
pack: manifest.name,
|
||||
type: r.type,
|
||||
hint: `gbrain schema add-type '${safeCliToken(r.type) ?? '<type>'}' OR retype the pages to a canonical type`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
};
|
||||
|
||||
/** All rules. File-plane callers can compose a subset via FILE_PLANE_RULES. */
|
||||
export const ALL_LINT_RULES: ReadonlyArray<{ name: string; rule: LintRule; planeAware: boolean }> = [
|
||||
{ name: 'alias_shadows_type', rule: aliasShadowsType, planeAware: false },
|
||||
@@ -362,6 +434,8 @@ export const ALL_LINT_RULES: ReadonlyArray<{ name: string; rule: LintRule; plane
|
||||
{ name: 'link_regex_catastrophic_backtrack', rule: linkRegexCatastrophicBacktrack, planeAware: false },
|
||||
{ name: 'extractable_empty_corpus', rule: extractableEmptyCorpus, planeAware: true },
|
||||
{ name: 'mutation_count_anomaly', rule: mutationCountAnomaly, planeAware: true },
|
||||
{ name: 'stored_type_is_alias', rule: storedTypeIsAlias, planeAware: true },
|
||||
{ name: 'stored_type_undeclared', rule: storedTypeUndeclared, planeAware: true },
|
||||
];
|
||||
|
||||
/** File-plane subset: rules safe to run inside `withMutation`'s pre-write gate. */
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Stored-type classification against the active schema pack.
|
||||
*
|
||||
* Motivation (alias-footgun incident class): an agent writes an explicit
|
||||
* frontmatter `type:` that is either an ALIAS of a canonical type or entirely
|
||||
* UNDECLARED in the pack. gbrain stores the string literally and never
|
||||
* re-normalizes it (import-file.ts type preservation), so the misrouting is
|
||||
* silent — different agents file the same concept under different types and
|
||||
* directories, and nobody notices until retrieval quality degrades. gbrain
|
||||
* cannot stop agent-side filing decisions, but it CAN make every non-canonical
|
||||
* explicit type loud at ingest (sync/import summary + SyncResult) and at audit
|
||||
* (`gbrain schema lint` data-plane rules).
|
||||
*
|
||||
* The pack parameter is structural (name + aliases + path_prefixes only) so
|
||||
* import-file can pass its thin activePack shape; the full SchemaPackManifest
|
||||
* satisfies it too.
|
||||
*/
|
||||
|
||||
export interface TypeUsagePack {
|
||||
page_types: ReadonlyArray<{
|
||||
name: string;
|
||||
path_prefixes: ReadonlyArray<string>;
|
||||
aliases?: ReadonlyArray<string>;
|
||||
}>;
|
||||
}
|
||||
|
||||
export type StoredTypeClass =
|
||||
| { kind: 'canonical' }
|
||||
| { kind: 'alias_of'; canonical: string; directory?: string }
|
||||
| { kind: 'undeclared' };
|
||||
|
||||
/**
|
||||
* Classify an explicit stored type against the pack: canonical page_type name,
|
||||
* alias of one (reports the canonical type + its filing directory =
|
||||
* path_prefixes[0]), or undeclared.
|
||||
*/
|
||||
export function classifyStoredType(type: string, pack: TypeUsagePack): StoredTypeClass {
|
||||
for (const pt of pack.page_types) {
|
||||
if (pt.name === type) return { kind: 'canonical' };
|
||||
}
|
||||
for (const pt of pack.page_types) {
|
||||
if (pt.aliases && pt.aliases.includes(type)) {
|
||||
return {
|
||||
kind: 'alias_of',
|
||||
canonical: pt.name,
|
||||
directory: pt.path_prefixes[0],
|
||||
};
|
||||
}
|
||||
}
|
||||
return { kind: 'undeclared' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Type strings come from user/agent frontmatter and get echoed into terminal
|
||||
* warnings — strip control/non-printable characters (ANSI-escape hygiene) and
|
||||
* cap the length so a hostile or garbled value can't mangle the terminal.
|
||||
*/
|
||||
export function sanitizeTypeForDisplay(type: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const cleaned = type.replace(/[\x00-\x1f\x7f]/g, '');
|
||||
return cleaned.length > 64 ? `${cleaned.slice(0, 61)}...` : cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stricter gate for embedding a type into a COPY-PASTEABLE command hint
|
||||
* (`gbrain schema add-type '<t>'`): display sanitization keeps quotes and
|
||||
* shell metacharacters, so a type like `x'; touch pwn; #` would escape the
|
||||
* quoting when an operator pastes the hint. Returns the value only when it
|
||||
* is a plain slug-safe token; callers render a placeholder otherwise.
|
||||
*/
|
||||
export function safeCliToken(v: string): string | null {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(v) ? v : null;
|
||||
}
|
||||
|
||||
/** One aggregated warning bucket per distinct non-canonical type. */
|
||||
export interface TypeWarningCount {
|
||||
kind: 'alias_of' | 'undeclared';
|
||||
type: string;
|
||||
canonical?: string;
|
||||
directory?: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the once-per-type-per-run summary lines for a sync/import run.
|
||||
* Returns [] when there is nothing to say.
|
||||
*/
|
||||
export function renderTypeWarningSummary(warnings: ReadonlyArray<TypeWarningCount>): string[] {
|
||||
const lines: string[] = [];
|
||||
for (const w of warnings) {
|
||||
const t = sanitizeTypeForDisplay(w.type);
|
||||
if (w.kind === 'alias_of') {
|
||||
const dir = w.directory ? ` (pages file under ${sanitizeTypeForDisplay(w.directory)})` : '';
|
||||
lines.push(
|
||||
`⚠ type '${t}' is an alias of '${sanitizeTypeForDisplay(w.canonical ?? '')}'${dir}: ` +
|
||||
`${w.count} file(s) used it explicitly — agents may route it inconsistently. ` +
|
||||
`Declare it as a page type or retype (gbrain schema --help).`,
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
`⚠ type '${t}' is not declared in the active schema pack: ${w.count} file(s) — ` +
|
||||
`stored as-is. Declare it (gbrain schema add-type) or use a canonical type.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
+62
-1
@@ -403,7 +403,63 @@ export type SyncableReason =
|
||||
| 'strategy'
|
||||
| 'pruned-dir'
|
||||
| 'include-glob-miss'
|
||||
| 'exclude-glob-hit';
|
||||
| 'exclude-glob-hit'
|
||||
| 'malformed-path';
|
||||
|
||||
/**
|
||||
* Path segments that can never be legitimate page filenames: square brackets
|
||||
* (the signature of markdown-link syntax leaking into a literal filename —
|
||||
* files named `[atoms/foo.md](https:/...)` were minted by misbehaving
|
||||
* producers and polluted search because slugifySegment STRIPS brackets
|
||||
* instead of rejecting them, yielding plausible-looking slugs) and ASCII
|
||||
* control characters. Parentheses are deliberately allowed — `meeting (1).md`
|
||||
* is a legitimate filename shape.
|
||||
*
|
||||
* Two-tier design (cross-model adversarial finding — both reviewers flagged
|
||||
* blanket-bracket collateral):
|
||||
* - ADMISSION (hasMalformedPathSegment): control chars reject on ANY path;
|
||||
* brackets reject only on MARKDOWN paths (.md/.mdx). Code-strategy lanes
|
||||
* keep indexing framework paths like `app/[id]/page.tsx`, which are
|
||||
* ubiquitous and legitimate.
|
||||
* - DESTRUCTION (isPoisonedPath): sync's row-DELETING lanes (reconcile,
|
||||
* modified-lane cleanup) act only on the actual injection signature —
|
||||
* `](` or control chars. A bare-bracket markdown file (`notes [draft].md`)
|
||||
* imported by a pre-gate release keeps its indexed row (it just can't
|
||||
* re-import until renamed; doctor's malformed_path_pages carries the
|
||||
* hint). Hard-deleting it on a routine post-upgrade full sync while the
|
||||
* file still exists would be silent data loss.
|
||||
*
|
||||
* IMPORTANT: these are PATH checks only. `](` inside file BODIES is normal
|
||||
* markdown and must never trip them.
|
||||
*/
|
||||
export const MALFORMED_PATH_SEGMENT_RE = /[\[\]\x00-\x1f]/;
|
||||
|
||||
/** The injection signature that marks a path as sweepable junk. */
|
||||
export const POISONED_PATH_RE = /\]\(|[\x00-\x1f]/;
|
||||
|
||||
/** Admission check: control chars anywhere; brackets on markdown paths. */
|
||||
export function hasMalformedPathSegment(path: string): boolean {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (/[\x00-\x1f]/.test(path)) return true;
|
||||
return /[\[\]]/.test(path) && /\.(md|mdx)$/i.test(path);
|
||||
}
|
||||
|
||||
/** Destruction gate: only paths matching the poison signature may have their DB rows swept. */
|
||||
export function isPoisonedPath(path: string): boolean {
|
||||
return POISONED_PATH_RE.test(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip control characters (and cap length) before echoing a malformed path
|
||||
* to a terminal — these paths contain control bytes BY DEFINITION, and a
|
||||
* crafted filename must not be able to inject ANSI escapes into sync output.
|
||||
* Brackets stay: they're printable and the informative part of the name.
|
||||
*/
|
||||
export function sanitizePathForDisplay(path: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const cleaned = path.replace(/[\x00-\x1f\x7f]/g, '\ufffd');
|
||||
return cleaned.length > 200 ? `${cleaned.slice(0, 197)}...` : cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical metafile basenames the markdown sync strategy intentionally
|
||||
@@ -438,6 +494,11 @@ function classifySync(path: string, opts: SyncableOptions = {}): SyncableReason
|
||||
|
||||
if (!isAllowedByStrategy(path, strategy)) return 'strategy';
|
||||
|
||||
// Reject filenames that can't be legitimate pages (bracket/control chars —
|
||||
// markdown-link syntax as a literal filename). Checked after `strategy` so
|
||||
// only files that would otherwise be admitted change classification.
|
||||
if (hasMalformedPathSegment(path)) return 'malformed-path';
|
||||
|
||||
// Skip every path segment that pruneDir would block walkers from descending
|
||||
// into. Catches hidden dirs (`.git`, `.obsidian`), `.raw/` sidecars, and
|
||||
// vendor/generated trees (`node_modules/`, `vendor/`, …) at any depth.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# gbrain agent workspace — template
|
||||
|
||||
<!-- gbrain-template-stamp: 0.46.10.0 -->
|
||||
<!-- gbrain-template-stamp: 0.46.11.0 -->
|
||||
|
||||
This repository is the **"Use this template"** distribution artifact for a
|
||||
[gbrain](https://github.com/garrytan/gbrain) personal-agent workspace — the same
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Direct unit tests for src/core/atomic-write.ts — previously pinned only
|
||||
* indirectly through the backlinks fixer. The properties that matter:
|
||||
*
|
||||
* 1. verify() throwing aborts the write: target byte-identical, tmp removed.
|
||||
* 2. Mode preservation past the umask: a 0o600 target stays 0o600 after an
|
||||
* atomic overwrite (open(2)'s mode arg is umask-masked; the explicit
|
||||
* chmod is the load-bearing line).
|
||||
* 3. verify() receives the ON-DISK bytes (not the in-memory candidate).
|
||||
* 4. No tmp residue on the happy path.
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync, readFileSync, readdirSync, statSync, chmodSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { atomicWriteFileSync } from '../src/core/atomic-write.ts';
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'atomic-write-test-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function tmpSiblings(): string[] {
|
||||
return readdirSync(dir).filter(f => f.includes('.tmp.'));
|
||||
}
|
||||
|
||||
describe('atomicWriteFileSync', () => {
|
||||
test('writes content and leaves no tmp residue', () => {
|
||||
const target = join(dir, 'page.md');
|
||||
atomicWriteFileSync(target, '# hello\n');
|
||||
expect(readFileSync(target, 'utf-8')).toBe('# hello\n');
|
||||
expect(tmpSiblings()).toEqual([]);
|
||||
});
|
||||
|
||||
test('verify() throw aborts: target untouched, tmp removed, error propagates', () => {
|
||||
const target = join(dir, 'page.md');
|
||||
writeFileSync(target, 'original\n');
|
||||
expect(() =>
|
||||
atomicWriteFileSync(target, 'candidate\n', {
|
||||
verify: () => { throw new Error('validation failed'); },
|
||||
}),
|
||||
).toThrow('validation failed');
|
||||
expect(readFileSync(target, 'utf-8')).toBe('original\n');
|
||||
expect(tmpSiblings()).toEqual([]);
|
||||
});
|
||||
|
||||
test('verify() receives the on-disk bytes of the tmp file', () => {
|
||||
const target = join(dir, 'page.md');
|
||||
const seen: string[] = [];
|
||||
atomicWriteFileSync(target, 'on-disk-check\n', {
|
||||
verify: (onDisk) => { seen.push(onDisk); },
|
||||
});
|
||||
expect(seen).toEqual(['on-disk-check\n']);
|
||||
});
|
||||
|
||||
test('preserves a restrictive target mode past the umask', () => {
|
||||
const target = join(dir, 'secret.md');
|
||||
writeFileSync(target, 'v1\n');
|
||||
chmodSync(target, 0o600);
|
||||
atomicWriteFileSync(target, 'v2\n');
|
||||
expect(readFileSync(target, 'utf-8')).toBe('v2\n');
|
||||
expect(statSync(target).mode & 0o7777).toBe(0o600);
|
||||
});
|
||||
|
||||
test('fresh file (no prior target) lands with default mode and content', () => {
|
||||
const target = join(dir, 'fresh.md');
|
||||
atomicWriteFileSync(target, 'fresh\n');
|
||||
expect(readFileSync(target, 'utf-8')).toBe('fresh\n');
|
||||
// Default 0o644 masked by whatever umask the test runs under — just
|
||||
// assert it is readable and not world-writable garbage.
|
||||
expect(statSync(target).mode & 0o200).toBe(0o200);
|
||||
});
|
||||
});
|
||||
@@ -106,6 +106,286 @@ describe('findBacklinkGaps dedupe (v0.36.x #967 regression)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// fixBacklinkGaps safety pipeline (frontmatter corruption incident regression)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, readdirSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
fixBacklinkGaps,
|
||||
insertTimelineEntry,
|
||||
findBacklinkGaps,
|
||||
type BacklinkGap,
|
||||
} from '../src/commands/backlinks.ts';
|
||||
import { frontmatterBodyOffset } from '../src/core/markdown.ts';
|
||||
import { acquirePageLock } from '../src/core/page-lock.ts';
|
||||
|
||||
const fence = '---';
|
||||
|
||||
function makeFixture(): { root: string; lockRoot: string; cleanup: () => void } {
|
||||
const root = mkdtempSync(join(tmpdir(), 'gbrain-backlinks-fix-'));
|
||||
const lockRoot = join(root, '.locks');
|
||||
mkdirSync(join(root, 'people'));
|
||||
mkdirSync(join(root, 'meetings'));
|
||||
return { root, lockRoot, cleanup: () => rmSync(root, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
function gapFor(target: string): BacklinkGap {
|
||||
return {
|
||||
sourcePage: 'meetings/standup.md',
|
||||
targetPage: target,
|
||||
entityName: 'Alice',
|
||||
sourceTitle: 'Standup',
|
||||
};
|
||||
}
|
||||
|
||||
describe('frontmatterBodyOffset', () => {
|
||||
test('no frontmatter → 0 (whole file is body)', () => {
|
||||
expect(frontmatterBodyOffset('# Alice\n\nBody.')).toBe(0);
|
||||
});
|
||||
|
||||
test('LF frontmatter → offset just after closing fence', () => {
|
||||
const content = `${fence}\ntype: person\n${fence}\n# Alice\n`;
|
||||
const off = frontmatterBodyOffset(content);
|
||||
expect(content.slice(off)).toBe('# Alice\n');
|
||||
});
|
||||
|
||||
test('CRLF fences count (trim semantics)', () => {
|
||||
const content = `${fence}\r\ntype: person\r\n${fence}\r\n# Alice\r\n`;
|
||||
const off = frontmatterBodyOffset(content);
|
||||
expect(content.slice(off)).toBe('# Alice\r\n');
|
||||
});
|
||||
|
||||
test('leading blank lines before the opener are allowed', () => {
|
||||
const content = `\n\n${fence}\ntype: person\n${fence}\nBody`;
|
||||
const off = frontmatterBodyOffset(content);
|
||||
expect(content.slice(off)).toBe('Body');
|
||||
});
|
||||
|
||||
test('unclosed fence → 0 (caller must pre-validate)', () => {
|
||||
expect(frontmatterBodyOffset(`${fence}\ntype: person\n# Alice`)).toBe(0);
|
||||
});
|
||||
|
||||
test('closing fence as final line without trailing newline', () => {
|
||||
const content = `${fence}\ntype: person\n${fence}`;
|
||||
expect(frontmatterBodyOffset(content)).toBe(content.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('insertTimelineEntry', () => {
|
||||
test('never anchors on a "## Timeline" string inside frontmatter', () => {
|
||||
// GUARD-DISTINGUISHING fixture (adversarial-review finding: a quoted
|
||||
// `description: "## Timeline"` is never at line start, so the ^-anchored
|
||||
// regex ignores it even WITHOUT the bodyStart slice — the old fixture
|
||||
// couldn't detect a broken guard). A line-start `## Timeline` INSIDE the
|
||||
// fence is valid YAML (a comment line) and matches the heading regex at
|
||||
// offset 0 — only the bodyStart slice keeps the insertion out of the
|
||||
// frontmatter.
|
||||
const content = `${fence}\ntype: person\n## Timeline\ntitle: Alice\n${fence}\n# Alice\n\nBody text.\n`;
|
||||
const bodyStart = frontmatterBodyOffset(content);
|
||||
expect(bodyStart).toBeGreaterThan(0);
|
||||
const out = insertTimelineEntry(content, bodyStart, '- new entry');
|
||||
// Frontmatter bytes untouched — a broken guard would have inserted the
|
||||
// entry into the YAML block right under the comment line.
|
||||
expect(out.slice(0, bodyStart)).toBe(content.slice(0, bodyStart));
|
||||
// No real body heading exists → a fresh section is appended at EOF.
|
||||
expect(out.trimEnd().endsWith('- new entry')).toBe(true);
|
||||
expect(out.indexOf('- new entry')).toBeGreaterThan(bodyStart);
|
||||
});
|
||||
|
||||
test('### Timeline and ## Timeline (2026) near-misses do not match; fresh section appended', () => {
|
||||
const content = `# Alice\n\n### Timeline\n\nsub\n\n## Timeline (2026)\n\nyear\n`;
|
||||
const out = insertTimelineEntry(content, 0, '- entry');
|
||||
expect(out).toContain('\n\n## Timeline\n\n- entry\n');
|
||||
// near-miss sections untouched
|
||||
expect(out).toContain('### Timeline\n\nsub');
|
||||
expect(out).toContain('## Timeline (2026)\n\nyear');
|
||||
});
|
||||
|
||||
test('two real ## Timeline headings → entry lands in the FIRST section', () => {
|
||||
const content = `# Alice\n\n## Timeline\n\n- first section\n\n## Notes\n\nx\n\n## Timeline\n\n- second section\n`;
|
||||
const out = insertTimelineEntry(content, 0, '- new');
|
||||
const firstIdx = out.indexOf('- new');
|
||||
expect(firstIdx).toBeGreaterThan(out.indexOf('- first section'));
|
||||
expect(firstIdx).toBeLessThan(out.indexOf('## Notes'));
|
||||
});
|
||||
|
||||
test('CRLF heading line matches', () => {
|
||||
const content = `# Alice\r\n\r\n## Timeline\r\n\r\n- old\r\n\r\n## Notes\r\nx\r\n`;
|
||||
const out = insertTimelineEntry(content, 0, '- new');
|
||||
expect(out.indexOf('- new')).toBeGreaterThan(out.indexOf('- old'));
|
||||
expect(out.indexOf('- new')).toBeLessThan(out.indexOf('## Notes'));
|
||||
});
|
||||
|
||||
test('heading present, no next section → appended at trimmed EOF', () => {
|
||||
const content = `# Alice\n\n## Timeline\n\n- old\n`;
|
||||
const out = insertTimelineEntry(content, 0, '- new');
|
||||
expect(out.endsWith('- old\n- new\n')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fixBacklinkGaps safety pipeline', () => {
|
||||
test('valid frontmatter page: entry inserted, frontmatter byte-identical, no tmp residue', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
const original = `${fence}\ntype: person\ntitle: Alice\n${fence}\n# Alice\n\n## Timeline\n\n- old\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), original);
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(1);
|
||||
expect(outcome.skipped).toHaveLength(0);
|
||||
const after = readFileSync(join(root, 'people/alice.md'), 'utf-8');
|
||||
const bodyStart = frontmatterBodyOffset(original);
|
||||
expect(after.slice(0, bodyStart)).toBe(original.slice(0, bodyStart));
|
||||
expect(after).toContain('Referenced in [Standup](../meetings/standup.md)');
|
||||
expect(readdirSync(join(root, 'people')).filter(f => f.includes('.tmp.'))).toHaveLength(0);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('frontmatter-less legacy page stays fixable (MISSING_OPEN is not a blocker)', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
writeFileSync(join(root, 'people/alice.md'), '# Alice\n');
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(1);
|
||||
expect(outcome.skipped).toHaveLength(0);
|
||||
const after = readFileSync(join(root, 'people/alice.md'), 'utf-8');
|
||||
expect(after.startsWith('# Alice')).toBe(true);
|
||||
expect(after).toContain('## Timeline');
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('pre-broken YAML fence (MISSING_CLOSE) → skipped, file byte-identical', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
const broken = `${fence}\ntype: person\n# Alice heading glued into frontmatter\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), broken);
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(0);
|
||||
expect(outcome.skipped).toHaveLength(1);
|
||||
expect(outcome.skipped[0].reason).toContain('MISSING_CLOSE');
|
||||
expect(readFileSync(join(root, 'people/alice.md'), 'utf-8')).toBe(broken);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('incident regression: bullet can never land above the frontmatter fence', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
// The incident shape: an entity page whose frontmatter carries a
|
||||
// LINE-START `## Timeline` (a valid YAML comment — the
|
||||
// guard-distinguishing form; a quoted mid-line mention can't detect a
|
||||
// broken bodyStart guard) and whose body has no Timeline section yet.
|
||||
const original = `${fence}\ntype: person\ntitle: Y Combinator\n## Timeline\nnotes: history below\n${fence}\n# Y Combinator\n\nBody text.\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), original);
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(1);
|
||||
const after = readFileSync(join(root, 'people/alice.md'), 'utf-8');
|
||||
// Byte 0 is still the opening fence; frontmatter intact.
|
||||
expect(after.startsWith(`${fence}\ntype: person`)).toBe(true);
|
||||
const bodyStart = frontmatterBodyOffset(original);
|
||||
expect(after.slice(0, bodyStart)).toBe(original.slice(0, bodyStart));
|
||||
// Entry landed in a fresh body section, below the fence.
|
||||
expect(after.indexOf('Referenced in')).toBeGreaterThan(bodyStart);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('dryRun counts fixes but writes nothing', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
const original = `${fence}\ntype: person\n${fence}\n# Alice\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), original);
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], true, { lockRoot });
|
||||
expect(outcome.fixed).toBe(1);
|
||||
expect(readFileSync(join(root, 'people/alice.md'), 'utf-8')).toBe(original);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('held page lock → that file skipped with a reason, others still fixed', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
writeFileSync(join(root, 'people/alice.md'), `${fence}\ntype: person\n${fence}\n# Alice\n`);
|
||||
writeFileSync(join(root, 'people/bob.md'), `${fence}\ntype: person\n${fence}\n# Bob\n`);
|
||||
const held = await acquirePageLock('people/alice', { lockRoot });
|
||||
expect(held).not.toBeNull();
|
||||
try {
|
||||
const outcome = await fixBacklinkGaps(
|
||||
root,
|
||||
[gapFor('people/alice.md'), gapFor('people/bob.md')],
|
||||
false,
|
||||
{ lockRoot },
|
||||
);
|
||||
expect(outcome.fixed).toBe(1); // bob only
|
||||
expect(outcome.skipped).toHaveLength(1);
|
||||
expect(outcome.skipped[0].page).toBe('people/alice.md');
|
||||
expect(outcome.skipped[0].reason).toContain('lock');
|
||||
} finally {
|
||||
await held!.release();
|
||||
}
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
test('frontmatter-only page (empty body) gets a section appended after the fence', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
const original = `${fence}\ntype: person\ntitle: Alice\n${fence}\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), original);
|
||||
const outcome = await fixBacklinkGaps(root, [gapFor('people/alice.md')], false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(1);
|
||||
const after = readFileSync(join(root, 'people/alice.md'), 'utf-8');
|
||||
const bodyStart = frontmatterBodyOffset(original);
|
||||
expect(after.slice(0, bodyStart)).toBe(original.slice(0, bodyStart));
|
||||
expect(after.slice(bodyStart)).toContain('## Timeline');
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('two gaps from different source pages batch into one target write: fixed=2, both bullets under Timeline', async () => {
|
||||
const { root, lockRoot, cleanup } = makeFixture();
|
||||
try {
|
||||
const original = `${fence}\ntype: person\ntitle: Alice\n${fence}\n# Alice\n\n## Timeline\n\n- old\n`;
|
||||
writeFileSync(join(root, 'people/alice.md'), original);
|
||||
const gaps: BacklinkGap[] = [
|
||||
{ sourcePage: 'meetings/standup.md', targetPage: 'people/alice.md', entityName: 'Alice', sourceTitle: 'Standup' },
|
||||
{ sourcePage: 'meetings/retro.md', targetPage: 'people/alice.md', entityName: 'Alice', sourceTitle: 'Retro' },
|
||||
];
|
||||
const outcome = await fixBacklinkGaps(root, gaps, false, { lockRoot });
|
||||
expect(outcome.fixed).toBe(2);
|
||||
expect(outcome.skipped).toHaveLength(0);
|
||||
const after = readFileSync(join(root, 'people/alice.md'), 'utf-8');
|
||||
// Frontmatter byte-identical.
|
||||
const bodyStart = frontmatterBodyOffset(original);
|
||||
expect(after.slice(0, bodyStart)).toBe(original.slice(0, bodyStart));
|
||||
// Both bullets present, and both land BELOW the Timeline heading.
|
||||
const headingIdx = after.indexOf('## Timeline');
|
||||
expect(headingIdx).toBeGreaterThan(bodyStart);
|
||||
const standupIdx = after.indexOf('Referenced in [Standup](../meetings/standup.md)');
|
||||
const retroIdx = after.indexOf('Referenced in [Retro](../meetings/retro.md)');
|
||||
expect(standupIdx).toBeGreaterThan(headingIdx);
|
||||
expect(retroIdx).toBeGreaterThan(headingIdx);
|
||||
// Exactly one Timeline section — the second gap must not mint a new one.
|
||||
expect(after.match(/^## Timeline$/gm)).toHaveLength(1);
|
||||
// No tmp residue from the atomic-write pipeline.
|
||||
expect(readdirSync(join(root, 'people')).filter(f => f.includes('.tmp.'))).toHaveLength(0);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseBacklinksArgs', () => {
|
||||
test('uses positional dir for check and fix subcommands', () => {
|
||||
expect(parseBacklinksArgs(['check', '/tmp/brain']).brainDir).toBe('/tmp/brain');
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Self-test for scripts/check-getpage-scoped-write.mjs — the
|
||||
* unscoped-check/scoped-write source-isolation guard.
|
||||
*
|
||||
* Runs the scanner against the COMMITTED fixtures at
|
||||
* test/fixtures/guards/check-getpage-scoped-write.mjs/{bad,good}/ (the same
|
||||
* fixtures guard-self-test.sh exercises), so the pinned shapes live in one
|
||||
* place. Also pins the file-scoped heuristic: an unscoped getPage in a file
|
||||
* with NO write path must not be flagged.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const SCRIPT = join(import.meta.dir, '..', 'scripts', 'check-getpage-scoped-write.mjs');
|
||||
const FIXTURES = join(import.meta.dir, 'fixtures', 'guards', 'check-getpage-scoped-write.mjs');
|
||||
|
||||
function runGuard(dir: string): { code: number; out: string } {
|
||||
const res = Bun.spawnSync([process.execPath, SCRIPT, dir]);
|
||||
return { code: res.exitCode, out: res.stderr.toString() + res.stdout.toString() };
|
||||
}
|
||||
|
||||
describe('check-getpage-scoped-write guard', () => {
|
||||
test('flags unscoped + ternary-undefined getPage reads in files that also write', () => {
|
||||
const { code, out } = runGuard(join(FIXTURES, 'bad'));
|
||||
expect(code).toBe(1);
|
||||
expect(out).toContain('fixture.ts:3'); // no-opts read
|
||||
expect(out).toContain('fixture.ts:8'); // conditional-undefined read (shorthand)
|
||||
expect(out).toContain('fixture.ts:14'); // EXPANDED ternary `{ sourceId: x } : undefined`
|
||||
expect(out).toContain('fixture.ts:19'); // empty-object false branch `: {}`
|
||||
expect(out).toContain("sourceId: x ?? 'default'");
|
||||
});
|
||||
|
||||
test('passes scoped reads, opt-out markers, and read-only files', () => {
|
||||
const { code, out } = runGuard(join(FIXTURES, 'good'));
|
||||
expect(code).toBe(0);
|
||||
expect(out).toContain('clean');
|
||||
});
|
||||
|
||||
test('the real src/ tree is clean (grandfathered allowlist is EMPTY by design)', () => {
|
||||
const res = Bun.spawnSync([process.execPath, SCRIPT], { cwd: join(import.meta.dir, '..') });
|
||||
const out = res.stderr.toString() + res.stdout.toString();
|
||||
expect(res.exitCode).toBe(0);
|
||||
expect(out).toContain('clean');
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -44,7 +44,7 @@ mock.module('../../src/commands/backlinks.ts', () => ({
|
||||
hasBacklink: () => false,
|
||||
buildBacklinkEntry: () => '',
|
||||
findBacklinkGaps: () => [],
|
||||
fixBacklinkGaps: () => 0,
|
||||
fixBacklinkGaps: async () => ({ fixed: 0, skipped: [] }),
|
||||
runBacklinks: async () => {},
|
||||
}));
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Five-issue fix wave — doctor's NEW queue_health surfaces (queue-type
|
||||
* divergence + waiting-TTL cancellation visibility) and the
|
||||
* malformed_path_pages discovery check.
|
||||
*
|
||||
* computeQueueHealthCheck is Postgres-only (short-circuits to ok on PGLite),
|
||||
* so its grouped SQL runs on a real PGLite engine behind a `kind: 'postgres'`
|
||||
* stub — the same harness as test/doctor-wedged-queue.test.ts.
|
||||
* malformed_path_pages has no exported compute function; it is driven through
|
||||
* buildChecks on the real PGLite engine (the seam
|
||||
* test/doctor-graph-coverage-soft-deleted.test.ts already uses).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { computeQueueHealthCheck, buildChecks } from '../src/commands/doctor.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
let base: PGLiteEngine;
|
||||
let pgLike: BrainEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
base = new PGLiteEngine();
|
||||
await base.connect({});
|
||||
await base.initSchema();
|
||||
// computeQueueHealthCheck only reads .kind + .executeRaw.
|
||||
pgLike = {
|
||||
kind: 'postgres',
|
||||
executeRaw: base.executeRaw.bind(base),
|
||||
} as unknown as BrainEngine;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await base.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(base);
|
||||
});
|
||||
|
||||
async function seed(
|
||||
queue: string,
|
||||
name: string,
|
||||
status: string,
|
||||
extra: { createdAtSql?: string; finishedAtSql?: string; errorText?: string } = {},
|
||||
): Promise<void> {
|
||||
await base.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, error_text, created_at, updated_at, finished_at)
|
||||
VALUES ($1, $2, $3, $4, ${extra.createdAtSql ?? 'now()'}, now(), ${extra.finishedAtSql ?? 'NULL'})`,
|
||||
[name, queue, status, extra.errorText ?? null],
|
||||
);
|
||||
}
|
||||
|
||||
describe('queue_health — DIVERGENT queue type detection', () => {
|
||||
it('flags a type whose intake structurally exceeds drain while a real backlog waits', async () => {
|
||||
// Lower the min-waiting knob (default 50) so a small seed triggers.
|
||||
await withEnv({ GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING: '5' }, async () => {
|
||||
// 6 waiting rows created within 24h: intake=6, completed=0, waiting=6.
|
||||
// waiting 6 > minWaiting 5 AND intake 6 > ratio 2 × max(0, 1) → DIVERGENT.
|
||||
// Stays under GBRAIN_QUEUE_WAITING_THRESHOLD (10) so the depth problem
|
||||
// can't fire and mask which surface produced the warn.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await seed('default', 'ingest-batch', 'waiting', {
|
||||
createdAtSql: "now() - interval '1 hour'",
|
||||
});
|
||||
}
|
||||
const check = await computeQueueHealthCheck(pgLike, {
|
||||
readWorkers: () => [{ queue: 'default' }],
|
||||
});
|
||||
expect(check.status).toBe('warn');
|
||||
expect(check.message).toContain("DIVERGENT queue type 'ingest-batch'");
|
||||
expect(check.message).toContain('intake 6/24h vs 0 completed/24h');
|
||||
// The quota-config admission hint must be paste-ready.
|
||||
expect(check.message).toContain('minions.quota_max_waiting.ingest-batch');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('queue_health — waiting-TTL cancellation visibility', () => {
|
||||
it('surfaces waiting_ttl_expired cancellations from the last 24h with the TTL tuning hint', async () => {
|
||||
for (const finishedAtSql of ["now() - interval '1 hour'", "now() - interval '2 hours'"]) {
|
||||
await seed('default', 'ingest-batch', 'cancelled', {
|
||||
errorText: 'waiting_ttl_expired: test',
|
||||
finishedAtSql,
|
||||
});
|
||||
}
|
||||
const check = await computeQueueHealthCheck(pgLike, { readWorkers: () => [] });
|
||||
expect(check.status).toBe('warn');
|
||||
expect(check.message).toContain("waiting-TTL cancelled 2 'ingest-batch' job(s)");
|
||||
expect(check.message).toContain('minions.ttl_waiting_hours.ingest-batch');
|
||||
});
|
||||
|
||||
it('ignores TTL cancellations older than the 24h window', async () => {
|
||||
await seed('default', 'ingest-batch', 'cancelled', {
|
||||
errorText: 'waiting_ttl_expired: test',
|
||||
finishedAtSql: "now() - interval '48 hours'",
|
||||
});
|
||||
const check = await computeQueueHealthCheck(pgLike, { readWorkers: () => [] });
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).not.toContain('waiting-TTL cancelled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('queue_health — healthy queue stays quiet', () => {
|
||||
it('neither the DIVERGENT nor the waiting-TTL string appears on a healthy seed', async () => {
|
||||
await seed('default', 'ingest-batch', 'waiting');
|
||||
await seed('default', 'ingest-batch', 'completed', { finishedAtSql: 'now()' });
|
||||
const check = await computeQueueHealthCheck(pgLike, {
|
||||
readWorkers: () => [{ queue: 'default' }],
|
||||
});
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).not.toContain('DIVERGENT');
|
||||
expect(check.message).not.toContain('waiting-TTL cancelled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('malformed_path_pages discovery check (buildChecks seam)', () => {
|
||||
it('warns naming the count + slug when a page is backed by a bracketed filename', async () => {
|
||||
await base.executeRaw(
|
||||
`INSERT INTO pages (source_id, slug, source_path, type, title, compiled_truth, timeline, frontmatter, content_hash)
|
||||
VALUES ('default', 'junk-1', '[foo.md](https-x).md', 'note', 'Junk', 'b', '', '{}'::jsonb, 'mp1')`,
|
||||
);
|
||||
const checks = await buildChecks(base, [], null);
|
||||
const check = checks.find((c) => c.name === 'malformed_path_pages');
|
||||
expect(check, 'malformed_path_pages check must be present').toBeDefined();
|
||||
expect(check!.status).toBe('warn');
|
||||
expect(check!.message).toContain('1 page(s) backed by malformed filenames');
|
||||
expect(check!.message).toContain('junk-1');
|
||||
});
|
||||
|
||||
it('clean DB → the check does not fire (absence is the ok state)', async () => {
|
||||
await base.executeRaw(
|
||||
`INSERT INTO pages (source_id, slug, source_path, type, title, compiled_truth, timeline, frontmatter, content_hash)
|
||||
VALUES ('default', 'clean-1', 'notes/clean-1.md', 'note', 'Clean', 'b', '', '{}'::jsonb, 'cl1')`,
|
||||
);
|
||||
const checks = await buildChecks(base, [], null);
|
||||
expect(checks.find((c) => c.name === 'malformed_path_pages')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1233,3 +1233,69 @@ describeBoth('Engine parity — ambient recall keyset + session cursor (v0.45.7)
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── unscoped getPage deterministic multi-source tiebreak ─────────────────
|
||||
// The pre-fix behavior: unscoped getPage was `LIMIT 1` with no ORDER BY, so a
|
||||
// slug present in several sources returned an ARBITRARY row (and an
|
||||
// existence-check + write pair could target different sources). Both engines
|
||||
// now pin `ORDER BY (source_id = 'default') DESC, source_id ASC` —
|
||||
// default-source first, then stable alpha. Parity here catches either engine
|
||||
// dropping the clause.
|
||||
describeBoth('Engine parity — unscoped getPage multi-source tiebreak', () => {
|
||||
let pgEngine: BrainEngine;
|
||||
let pgliteEngine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
pgEngine = await setupDB();
|
||||
pgliteEngine = new PGLiteEngine();
|
||||
await pgliteEngine.connect({});
|
||||
await pgliteEngine.initSchema();
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
for (const src of ['archive', 'work', 'zeta']) {
|
||||
await eng.executeRaw(
|
||||
`INSERT INTO sources (id, name, config) VALUES ($1, $1, '{}'::jsonb) ON CONFLICT (id) DO NOTHING`,
|
||||
[src],
|
||||
);
|
||||
}
|
||||
// Same slug in 'archive' AND 'default' — default must win even though
|
||||
// 'archive' sorts first alphabetically.
|
||||
await eng.putPage('tiebreak/with-default', {
|
||||
type: 'note', title: 'archive row', compiled_truth: 'a', timeline: '',
|
||||
}, { sourceId: 'archive' });
|
||||
await eng.putPage('tiebreak/with-default', {
|
||||
type: 'note', title: 'default row', compiled_truth: 'd', timeline: '',
|
||||
}, { sourceId: 'default' });
|
||||
// Same slug in 'work' AND 'zeta' only (no default row) — the
|
||||
// alphabetically-first source wins.
|
||||
await eng.putPage('tiebreak/no-default', {
|
||||
type: 'note', title: 'work row', compiled_truth: 'w', timeline: '',
|
||||
}, { sourceId: 'work' });
|
||||
await eng.putPage('tiebreak/no-default', {
|
||||
type: 'note', title: 'zeta row', compiled_truth: 'z', timeline: '',
|
||||
}, { sourceId: 'zeta' });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await pgliteEngine.disconnect();
|
||||
await teardownDB();
|
||||
}, 30_000);
|
||||
|
||||
test('unscoped getPage prefers the default-source row on both engines', async () => {
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
const page = await eng.getPage('tiebreak/with-default');
|
||||
expect(page).not.toBeNull();
|
||||
expect(page!.source_id).toBe('default');
|
||||
expect(page!.title).toBe('default row');
|
||||
}
|
||||
});
|
||||
|
||||
test('unscoped getPage falls back to the alphabetically-first source when no default row exists', async () => {
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
const page = await eng.getPage('tiebreak/no-default');
|
||||
expect(page).not.toBeNull();
|
||||
expect(page!.source_id).toBe('work');
|
||||
expect(page!.title).toBe('work row');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -301,3 +301,24 @@ describe('v0.42.43.0 #2095 — volunteer-events sink + cycle purge wiring (struc
|
||||
expect(src).toMatch(/purged_volunteer_events_count/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('five-issue fix wave — integrity progress is (source_id, slug)-keyed', () => {
|
||||
// integrity.ts's resume progress used to be keyed by slug alone, so a resume
|
||||
// SKIPPED same-slug pages in every other source (the scan iterates
|
||||
// (slug, source_id) pairs from listAllPageRefs). Behavioral coverage would
|
||||
// need live resolvers; the keying shape is what must not regress.
|
||||
test('integrity.ts keys seen/progress by progressKey(source_id, slug) and persists source_id', () => {
|
||||
const src = readFileSync('src/commands/integrity.ts', 'utf8');
|
||||
expect(src).toMatch(/function progressKey\(/);
|
||||
expect(src).toMatch(/seen\.has\(progressKey\(source_id, slug\)\)/);
|
||||
expect(src).toMatch(/seen\.add\(progressKey\(entry\.source_id, entry\.slug\)\)/);
|
||||
// Every appendProgress site persists the source_id.
|
||||
const appends = src.match(/appendProgress\(\{[^}]*\}\)/g) ?? [];
|
||||
expect(appends.length).toBeGreaterThan(0);
|
||||
for (const call of appends) {
|
||||
expect(call).toContain('source_id');
|
||||
}
|
||||
// One writer PER SOURCE (a single default-scoped writer was the bug).
|
||||
expect(src).toMatch(/new BrainWriter\(engine, \{ strictMode: 'off', sourceId \}\)/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// BAD: unscoped existence check + default-scoped write in one file.
|
||||
export async function badNoOpts(engine: any, slug: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug);
|
||||
if (!existing) await engine.putPage(slug, { title: 'x' });
|
||||
}
|
||||
// BAD: conditional-undefined read (any-source when unset) + write via importFromContent.
|
||||
export async function badTernary(engine: any, slug: string, sourceId?: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
if (!existing) await importFromContent(engine, slug, '# x', { sourceId });
|
||||
}
|
||||
// BAD: EXPANDED ternary — `{ sourceId: x }` object-literal colon must not hide the
|
||||
// undefined false branch (regression pin for the [^:]* regex gap).
|
||||
export async function badExpandedTernary(engine: any, slug: string, src?: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug, src ? { sourceId: src } : undefined);
|
||||
if (!existing) await engine.putPage(slug, { title: 'x' }, { sourceId: src ?? 'default' });
|
||||
}
|
||||
// BAD: empty-object false branch — {} is just as unscoped as undefined.
|
||||
export async function badEmptyObjectBranch(engine: any, slug: string, src?: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug, src ? { sourceId: src } : {});
|
||||
if (!existing) await engine.putPage(slug, { title: 'x' }, { sourceId: src ?? 'default' });
|
||||
}
|
||||
declare function importFromContent(...args: unknown[]): Promise<unknown>;
|
||||
@@ -0,0 +1,16 @@
|
||||
// GOOD: read scoped to the write's source (the canonical pattern).
|
||||
export async function goodScoped(engine: any, slug: string, sourceId?: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
|
||||
if (!existing) await engine.putPage(slug, { title: 'x' }, { sourceId: sourceId ?? 'default' });
|
||||
}
|
||||
// GOOD: unscoped read with an explicit opt-out marker (read-only first-match semantics).
|
||||
export async function goodMarked(engine: any, slug: string): Promise<void> {
|
||||
const page = await engine.getPage(slug); // gbrain-allow-unscoped-getpage: read-only probe, first-match semantics documented
|
||||
if (page) await engine.putPage(slug, { title: page.title }, { sourceId: 'default' });
|
||||
}
|
||||
// GOOD: inner ternary INSIDE a scoped opts object — always source-scoped, must
|
||||
// not false-positive on the "ends in undefined branch" probe.
|
||||
export async function goodInnerTernary(engine: any, slug: string, src?: string): Promise<void> {
|
||||
const existing = await engine.getPage(slug, { sourceId: src ? src : 'default' });
|
||||
if (!existing) await engine.putPage(slug, { title: 'x' }, { sourceId: src ?? 'default' });
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// GOOD: file has no write path at all — unscoped first-match reads are allowed.
|
||||
export async function readOnlyProbe(engine: any, slug: string): Promise<boolean> {
|
||||
const page = await engine.getPage(slug);
|
||||
return page !== null;
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdirSync, writeFileSync, symlinkSync, rmSync, mkdtempSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { collectMarkdownFiles } from '../src/commands/import.ts';
|
||||
import { collectMarkdownFiles, collectSyncableFiles } from '../src/commands/import.ts';
|
||||
|
||||
// These tests exercise the filesystem walker that feeds `gbrain import`.
|
||||
// They target L002 (report/findings.md): a malicious symlink inside a shared
|
||||
@@ -87,3 +88,47 @@ describe('collectMarkdownFiles — symlink containment', () => {
|
||||
expect(files).not.toContain(join(root, 'dangling.md'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('collectSyncableFiles — malformed-filename exclusion (both routes)', () => {
|
||||
let root: string;
|
||||
const JUNK = '[foo.md](https-example).md';
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), 'gbrain-walker-malformed-'));
|
||||
writeFileSync(join(root, 'legit.md'), '# legit\n');
|
||||
writeFileSync(join(root, JUNK), '# junk\n');
|
||||
mkdirSync(join(root, 'notes'));
|
||||
writeFileSync(join(root, 'notes', '[wip] draft.md'), '# wip\n');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('FS-walk route excludes bracket-named files', () => {
|
||||
const files = collectSyncableFiles(root);
|
||||
expect(files).toContain(join(root, 'legit.md'));
|
||||
expect(files).not.toContain(join(root, JUNK));
|
||||
expect(files).not.toContain(join(root, 'notes', '[wip] draft.md'));
|
||||
});
|
||||
|
||||
test('FS-walk route never DESCENDS into a bracket-named directory (clean children stay out)', () => {
|
||||
// The descent-time check is a separate carve-out from the per-file check:
|
||||
// a clean-named .md INSIDE a bracket-named dir still has a malformed PATH.
|
||||
mkdirSync(join(root, '[archive] 2026'));
|
||||
writeFileSync(join(root, '[archive] 2026', 'clean-name.md'), '# hidden by dir\n');
|
||||
const files = collectSyncableFiles(root);
|
||||
expect(files).toContain(join(root, 'legit.md'));
|
||||
expect(files).not.toContain(join(root, '[archive] 2026', 'clean-name.md'));
|
||||
});
|
||||
|
||||
test('git fast-path route excludes bracket-named files', () => {
|
||||
execSync('git init', { cwd: root, stdio: 'pipe' });
|
||||
execSync('git config user.email "t@t.com" && git config user.name "T"', { cwd: root, stdio: 'pipe' });
|
||||
execSync('git add -A && git commit -m x', { cwd: root, stdio: 'pipe' });
|
||||
const files = collectSyncableFiles(root);
|
||||
expect(files).toContain(join(root, 'legit.md'));
|
||||
expect(files).not.toContain(join(root, JUNK));
|
||||
expect(files).not.toContain(join(root, 'notes', '[wip] draft.md'));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* `jobs stats` divergence + waiting-TTL surfaces — WIRING test (five-issue
|
||||
* fix wave, commit 5). The data source (getStats drained fields + waiting_now)
|
||||
* is pinned in test/minions-admission.test.ts; this covers the stats-case
|
||||
* composition:
|
||||
* 1. ⚠ DIVERGENT QUEUE fires when intake > ratio × completed AND waiting
|
||||
* depth exceeds the floor — computed against COMPLETED drain, so a
|
||||
* TTL-cancellation storm cannot masquerade as healthy throughput.
|
||||
* 2. The scream carries the quota opt-in hint (default protection layer —
|
||||
* the quota itself ships config-only per user decision D2C).
|
||||
* 3. ⚠ Waiting-TTL line reports 24h sweep cancellations by name.
|
||||
* 4. --json emits a parseable document with divergent + ttl_cancelled_24h.
|
||||
* 5. Healthy queues (drain keeping up) render neither scream.
|
||||
*
|
||||
* Serial: mutates GBRAIN_QUEUE_DIVERGENCE_* env via withEnv and captures
|
||||
* console.log around runJobs.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runJobs } from '../src/commands/jobs.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM minion_jobs`, []);
|
||||
});
|
||||
|
||||
async function seedJobs(opts: { waiting: number; completed: number; ttlCancelled?: number }): Promise<void> {
|
||||
for (let i = 0; i < opts.waiting; i++) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, data) VALUES ('subagent', 'default', 'waiting', '{}'::jsonb)`,
|
||||
[],
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < opts.completed; i++) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, data, started_at, finished_at)
|
||||
VALUES ('subagent', 'default', 'completed', '{}'::jsonb, now() - interval '10 minutes', now() - interval '5 minutes')`,
|
||||
[],
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < (opts.ttlCancelled ?? 0); i++) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, data, error_text, finished_at)
|
||||
VALUES ('subagent', 'default', 'cancelled', '{}'::jsonb,
|
||||
'waiting_ttl_expired: waited > 48h in queue', now() - interval '1 hour')`,
|
||||
[],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureStats(args: string[]): Promise<string> {
|
||||
const origLog = console.log;
|
||||
let out = '';
|
||||
console.log = (...a: unknown[]) => { out += a.map(String).join(' ') + '\n'; };
|
||||
try {
|
||||
await withEnv(
|
||||
{ GBRAIN_QUEUE_DIVERGENCE_RATIO: '2', GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING: '5' },
|
||||
async () => { await runJobs(engine, args); },
|
||||
);
|
||||
} finally {
|
||||
console.log = origLog;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('jobs stats — DIVERGENT QUEUE + waiting-TTL screams', () => {
|
||||
test('divergent intake vs completed drain screams with the quota opt-in hint', async () => {
|
||||
await seedJobs({ waiting: 20, completed: 2, ttlCancelled: 6 });
|
||||
const out = await captureStats(['stats']);
|
||||
expect(out).toContain("DIVERGENT QUEUE type 'subagent'");
|
||||
expect(out).toContain('gbrain config set minions.quota_max_waiting.subagent');
|
||||
// TTL sweep visibility rides the divergence report + its own line.
|
||||
expect(out).toContain('Waiting-TTL cancelled 6 job(s) in the last 24h');
|
||||
expect(out).toContain('minions.ttl_waiting_hours');
|
||||
});
|
||||
|
||||
test('TTL cancellations do NOT count as drain (the shredder cannot look healthy)', async () => {
|
||||
// GUARD-DISTINGUISHING seed (adversarial-review finding: the old
|
||||
// 20/0/15 seed screamed under BOTH metrics, so the test couldn't detect
|
||||
// a naive-drain regression). Here: intake = 20 waiting + 30 cancelled =
|
||||
// 50 created-in-window; a NAIVE drain metric (completed+cancelled = 30)
|
||||
// gives 50 <= 2×30 → silent; the completed-based metric (0 completed)
|
||||
// gives 50 > 2×1 → screams. Only the correct metric fires.
|
||||
await seedJobs({ waiting: 20, completed: 0, ttlCancelled: 30 });
|
||||
const out = await captureStats(['stats']);
|
||||
expect(out).toContain('DIVERGENT QUEUE');
|
||||
expect(out).toContain('backlog never drains at current rate');
|
||||
});
|
||||
|
||||
test('healthy queue (drain keeps up) renders no screams', async () => {
|
||||
await seedJobs({ waiting: 2, completed: 40 });
|
||||
const out = await captureStats(['stats']);
|
||||
expect(out).not.toContain('DIVERGENT QUEUE');
|
||||
expect(out).not.toContain('Waiting-TTL cancelled');
|
||||
});
|
||||
|
||||
test('--json emits parseable divergent + ttl_cancelled_24h + per-type drain fields', async () => {
|
||||
await seedJobs({ waiting: 20, completed: 2, ttlCancelled: 3 });
|
||||
const out = await captureStats(['stats', '--json']);
|
||||
const doc = JSON.parse(out);
|
||||
expect(Array.isArray(doc.divergent)).toBe(true);
|
||||
expect(doc.divergent[0].name).toBe('subagent');
|
||||
expect(doc.divergent[0].waiting_now).toBe(20);
|
||||
expect(doc.ttl_cancelled_24h).toEqual([{ name: 'subagent', count: 3 }]);
|
||||
const sub = doc.by_type.find((t: { name: string }) => t.name === 'subagent');
|
||||
expect(sub.drained_completed).toBe(2);
|
||||
expect(sub.drained_cancelled).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,430 @@
|
||||
/**
|
||||
* Queue admission control (five-issue fix wave, commit 5): param-coalescing,
|
||||
* waiting-TTL sweep, and the name-global quota. Covers the four
|
||||
* outside-voice-hardened invariants:
|
||||
* - owner lanes never cross (hash INCLUDES __owner_client_id),
|
||||
* - parented submits never coalesce (aggregator bookkeeping),
|
||||
* - coalescing is age-bounded to ttl/2 (a fresh submit must not coalesce
|
||||
* onto a nearly-expired row and die an hour later),
|
||||
* - TTL cancellation flows through cancelJobs (descendants + child_done +
|
||||
* parent resolution) and writes the reason error_text every alerting
|
||||
* surface keys on.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
import {
|
||||
computeParamHash,
|
||||
QueueQuotaExceededError,
|
||||
runWaitingTtlTick,
|
||||
TTL_NOTICE_SHOWN_KEY,
|
||||
_resetAdmissionCacheForTest,
|
||||
} from '../src/core/minions/admission.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let queue: MinionQueue;
|
||||
|
||||
const SUB = { allowProtectedSubmit: true } as const;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
queue = new MinionQueue(engine);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
// Targeted reset (same posture as the other minion suites): wipe jobs +
|
||||
// inbox + this suite's config keys, but keep the schema-version row that
|
||||
// ensureSchema() checks.
|
||||
await engine.executeRaw(`DELETE FROM minion_inbox`, []);
|
||||
await engine.executeRaw(`DELETE FROM minion_jobs`, []);
|
||||
await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'minions.%'`, []);
|
||||
_resetAdmissionCacheForTest();
|
||||
});
|
||||
|
||||
async function backdate(id: number, hours: number): Promise<void> {
|
||||
// Both timestamps: the TTL sweep + coalesce age-bound key on updated_at
|
||||
// (a requeue refreshes the TTL window); getStats oldest-wait uses created_at.
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET created_at = now() - ($2 * interval '1 hour'),
|
||||
updated_at = now() - ($2 * interval '1 hour') WHERE id = $1`,
|
||||
[id, hours],
|
||||
);
|
||||
}
|
||||
|
||||
describe('computeParamHash', () => {
|
||||
test('stable across key order, excludes only __param_hash', () => {
|
||||
const a = computeParamHash({ prompt: 'p', model: 'm', nested: { x: 1, y: 2 } });
|
||||
const b = computeParamHash({ model: 'm', nested: { y: 2, x: 1 }, prompt: 'p' });
|
||||
expect(a).toBe(b);
|
||||
expect(computeParamHash({ prompt: 'p', __param_hash: 'stale' })).toBe(computeParamHash({ prompt: 'p' }));
|
||||
});
|
||||
|
||||
test('__owner_client_id is INCLUDED — owner lanes hash differently', () => {
|
||||
const a = computeParamHash({ prompt: 'p', __owner_client_id: 'client-a' });
|
||||
const b = computeParamHash({ prompt: 'p', __owner_client_id: 'client-b' });
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('param-coalescing (subagent default-on)', () => {
|
||||
test('identical parentless submit coalesces onto the waiting row', async () => {
|
||||
const first = await queue.add('subagent', { prompt: 'daily digest', model: 'anthropic:claude-sonnet-5' }, {}, SUB);
|
||||
expect(first.coalesced).not.toBe(true);
|
||||
expect((first.data as Record<string, unknown>).__param_hash).toBeDefined();
|
||||
|
||||
const second = await queue.add('subagent', { prompt: 'daily digest', model: 'anthropic:claude-sonnet-5' }, {}, SUB);
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(second.coalesced).toBe(true);
|
||||
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs WHERE name = 'subagent'`,
|
||||
);
|
||||
expect(parseInt(rows[0].count, 10)).toBe(1);
|
||||
});
|
||||
|
||||
test('different prompts insert separate rows', async () => {
|
||||
const a = await queue.add('subagent', { prompt: 'task A' }, {}, SUB);
|
||||
const b = await queue.add('subagent', { prompt: 'task B' }, {}, SUB);
|
||||
expect(b.id).not.toBe(a.id);
|
||||
expect(b.coalesced).not.toBe(true);
|
||||
});
|
||||
|
||||
test('identical prompts from different owner lanes never coalesce', async () => {
|
||||
const a = await queue.add('subagent', { prompt: 'same', __owner_client_id: 'client-a' }, {}, SUB);
|
||||
const b = await queue.add('subagent', { prompt: 'same', __owner_client_id: 'client-b' }, {}, SUB);
|
||||
expect(b.id).not.toBe(a.id);
|
||||
});
|
||||
|
||||
test('a RUNNING identical job does not suppress a re-run (waiting-only)', async () => {
|
||||
const first = await queue.add('subagent', { prompt: 'rerun me' }, {}, SUB);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'active', lock_until = now() + interval '5 minutes' WHERE id = $1`,
|
||||
[first.id],
|
||||
);
|
||||
const second = await queue.add('subagent', { prompt: 'rerun me' }, {}, SUB);
|
||||
expect(second.id).not.toBe(first.id);
|
||||
});
|
||||
|
||||
test('EMPTY payloads never coalesce (no dedupe signal — scaffolding/placeholder jobs stay distinct)', async () => {
|
||||
const a = await queue.add('subagent', {}, {}, SUB);
|
||||
const b = await queue.add('subagent', {}, {}, SUB);
|
||||
expect(b.id).not.toBe(a.id);
|
||||
expect((a.data as Record<string, unknown>).__param_hash).toBeUndefined();
|
||||
});
|
||||
|
||||
test('parented submits never coalesce (fanout children belong to their parent)', async () => {
|
||||
const parent = await queue.add('subagent_aggregator', { kind: 'agg' }, {}, SUB);
|
||||
const c1 = await queue.add('subagent', { prompt: 'chunk' }, { parent_job_id: parent.id }, SUB);
|
||||
const c2 = await queue.add('subagent', { prompt: 'chunk' }, { parent_job_id: parent.id }, SUB);
|
||||
expect(c2.id).not.toBe(c1.id);
|
||||
});
|
||||
|
||||
test('age-bounded: a near-TTL waiting row is not a coalesce target', async () => {
|
||||
const first = await queue.add('subagent', { prompt: 'old intent' }, {}, SUB);
|
||||
await backdate(first.id, 47); // TTL default 48h → bound is 24h
|
||||
const second = await queue.add('subagent', { prompt: 'old intent' }, {}, SUB);
|
||||
expect(second.id).not.toBe(first.id);
|
||||
});
|
||||
|
||||
test('kill-switch GBRAIN_MINIONS_ADMISSION=0 disables coalescing', async () => {
|
||||
await withEnv({ GBRAIN_MINIONS_ADMISSION: '0' }, async () => {
|
||||
_resetAdmissionCacheForTest();
|
||||
const a = await queue.add('subagent', { prompt: 'killed' }, {}, SUB);
|
||||
const b = await queue.add('subagent', { prompt: 'killed' }, {}, SUB);
|
||||
expect(b.id).not.toBe(a.id);
|
||||
});
|
||||
_resetAdmissionCacheForTest(); // don't leak the killed-policy cache to later tests
|
||||
});
|
||||
|
||||
test('config off-switch minions.coalesce_params.subagent=false disables it', async () => {
|
||||
await engine.setConfig('minions.coalesce_params.subagent', 'false');
|
||||
_resetAdmissionCacheForTest();
|
||||
const a = await queue.add('subagent', { prompt: 'cfg-off' }, {}, SUB);
|
||||
const b = await queue.add('subagent', { prompt: 'cfg-off' }, {}, SUB);
|
||||
expect(b.id).not.toBe(a.id);
|
||||
});
|
||||
|
||||
test('opt-in for a non-default name via coalesce_params submit option', async () => {
|
||||
const a = await queue.add('shell', { command: 'echo hi' }, { coalesce_params: true }, SUB);
|
||||
const b = await queue.add('shell', { command: 'echo hi' }, { coalesce_params: true }, SUB);
|
||||
expect(b.id).toBe(a.id);
|
||||
expect(b.coalesced).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('name-global quota (config-only, no shipped default)', () => {
|
||||
test('no quota configured → unlimited (D2C: default protection is the scream, not the cap)', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await queue.add('subagent', { prompt: `distinct ${i}` }, {}, SUB);
|
||||
}
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs WHERE name = 'subagent' AND status = 'waiting'`,
|
||||
);
|
||||
expect(parseInt(rows[0].count, 10)).toBe(5);
|
||||
});
|
||||
|
||||
test('configured quota rejects with a typed, actionable error', async () => {
|
||||
await engine.setConfig('minions.quota_max_waiting.subagent', '2');
|
||||
_resetAdmissionCacheForTest();
|
||||
await queue.add('subagent', { prompt: 'q1' }, {}, SUB);
|
||||
await queue.add('subagent', { prompt: 'q2' }, {}, SUB);
|
||||
let threw: unknown = null;
|
||||
try {
|
||||
await queue.add('subagent', { prompt: 'q3' }, {}, SUB);
|
||||
} catch (e) {
|
||||
threw = e;
|
||||
}
|
||||
expect(threw).toBeInstanceOf(QueueQuotaExceededError);
|
||||
expect((threw as Error).message).toContain('minions.quota_max_waiting.subagent');
|
||||
expect((threw as QueueQuotaExceededError).code).toBe('quota_exceeded');
|
||||
});
|
||||
|
||||
test('quota counts the name across ALL queues (private fanout queues cannot dodge it)', async () => {
|
||||
await engine.setConfig('minions.quota_max_waiting.subagent', '2');
|
||||
_resetAdmissionCacheForTest();
|
||||
await queue.add('subagent', { prompt: 'a' }, { queue: 'dream-inline-1' }, SUB);
|
||||
await queue.add('subagent', { prompt: 'b' }, { queue: 'dream-inline-2' }, SUB);
|
||||
await expect(
|
||||
queue.add('subagent', { prompt: 'c' }, { queue: 'dream-inline-3' }, SUB),
|
||||
).rejects.toThrow('quota');
|
||||
});
|
||||
|
||||
test('an identical submit coalesces BEFORE the quota check (dedupe is not rejection)', async () => {
|
||||
await engine.setConfig('minions.quota_max_waiting.subagent', '2');
|
||||
_resetAdmissionCacheForTest();
|
||||
const a = await queue.add('subagent', { prompt: 'same one' }, {}, SUB);
|
||||
await queue.add('subagent', { prompt: 'other' }, {}, SUB);
|
||||
// At quota. An IDENTICAL re-submit must return the existing row, not throw.
|
||||
const again = await queue.add('subagent', { prompt: 'same one' }, {}, SUB);
|
||||
expect(again.id).toBe(a.id);
|
||||
expect(again.coalesced).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('waiting-TTL sweep (handleWaitingTTL via cancelJobs)', () => {
|
||||
test('cancels waiting rows past TTL with the reason error_text; young rows untouched', async () => {
|
||||
const old = await queue.add('subagent', { prompt: 'ancient' }, {}, SUB);
|
||||
const young = await queue.add('subagent', { prompt: 'fresh' }, {}, SUB);
|
||||
await backdate(old.id, 49);
|
||||
|
||||
const { cancelled, by_name } = await queue.handleWaitingTTL();
|
||||
expect(cancelled).toBe(1);
|
||||
expect(by_name.subagent).toBe(1);
|
||||
|
||||
const oldRow = await queue.getJob(old.id);
|
||||
expect(oldRow?.status).toBe('cancelled');
|
||||
expect(oldRow?.error_text).toContain('waiting_ttl_expired');
|
||||
expect(oldRow?.finished_at).toBeTruthy();
|
||||
const youngRow = await queue.getJob(young.id);
|
||||
expect(youngRow?.status).toBe('waiting');
|
||||
});
|
||||
|
||||
test('active and delayed rows are never TTL-cancelled', async () => {
|
||||
const active = await queue.add('subagent', { prompt: 'running' }, {}, SUB);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'active', lock_until = now() + interval '5 minutes',
|
||||
created_at = now() - interval '100 hours' WHERE id = $1`,
|
||||
[active.id],
|
||||
);
|
||||
const { cancelled } = await queue.handleWaitingTTL();
|
||||
expect(cancelled).toBe(0);
|
||||
expect((await queue.getJob(active.id))?.status).toBe('active');
|
||||
});
|
||||
|
||||
test('CRITICAL: TTL-cancelling a fanout child unblocks its waiting-children parent', async () => {
|
||||
const parent = await queue.add('subagent_aggregator', { kind: 'agg' }, {}, SUB);
|
||||
const child = await queue.add(
|
||||
'subagent',
|
||||
{ prompt: 'doomed child' },
|
||||
{ parent_job_id: parent.id, on_child_fail: 'continue' },
|
||||
SUB,
|
||||
);
|
||||
expect((await queue.getJob(parent.id))?.status).toBe('waiting-children');
|
||||
await backdate(child.id, 49);
|
||||
|
||||
const { cancelled } = await queue.handleWaitingTTL();
|
||||
expect(cancelled).toBe(1);
|
||||
expect((await queue.getJob(child.id))?.status).toBe('cancelled');
|
||||
// The bookkeeping cancelJobs carries: parent resolves out of
|
||||
// waiting-children instead of wedging forever (the v128-raw-UPDATE trap).
|
||||
expect((await queue.getJob(parent.id))?.status).toBe('waiting');
|
||||
// And the child_done inbox message landed for the parent.
|
||||
const inbox = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_inbox WHERE job_id = $1`,
|
||||
[parent.id],
|
||||
);
|
||||
expect(parseInt(inbox[0].count, 10)).toBe(1);
|
||||
});
|
||||
|
||||
test('per-tick cap bounds one sweep; the backlog drains across ticks oldest-first', async () => {
|
||||
const ids: number[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const j = await queue.add('subagent', { prompt: `stale ${i}` }, {}, SUB);
|
||||
await backdate(j.id, 50 + i);
|
||||
ids.push(j.id);
|
||||
}
|
||||
const tick1 = await queue.handleWaitingTTL({ maxPerTick: 2 });
|
||||
expect(tick1.cancelled).toBe(2);
|
||||
const tick2 = await queue.handleWaitingTTL({ maxPerTick: 2 });
|
||||
expect(tick2.cancelled).toBe(2);
|
||||
const tick3 = await queue.handleWaitingTTL({ maxPerTick: 2 });
|
||||
expect(tick3.cancelled).toBe(1);
|
||||
for (const id of ids) {
|
||||
expect((await queue.getJob(id))?.status).toBe('cancelled');
|
||||
}
|
||||
});
|
||||
|
||||
test('minions.ttl_waiting_hours.subagent=0 disables the sweep for that name', async () => {
|
||||
await engine.setConfig('minions.ttl_waiting_hours.subagent', '0');
|
||||
const j = await queue.add('subagent', { prompt: 'protected' }, {}, SUB);
|
||||
await backdate(j.id, 500);
|
||||
const { cancelled } = await queue.handleWaitingTTL();
|
||||
expect(cancelled).toBe(0);
|
||||
expect((await queue.getJob(j.id))?.status).toBe('waiting');
|
||||
});
|
||||
|
||||
test('config override listConfigKeys path: TTL applies to a non-default name', async () => {
|
||||
await engine.setConfig('minions.ttl_waiting_hours.shell', '1');
|
||||
const j = await queue.add('shell', { command: 'echo stale' }, {}, SUB);
|
||||
await backdate(j.id, 2);
|
||||
const { by_name } = await queue.handleWaitingTTL();
|
||||
expect(by_name.shell).toBe(1);
|
||||
expect((await queue.getJob(j.id))?.status).toBe('cancelled');
|
||||
});
|
||||
|
||||
test('a TTL-cancelled row frees its idempotency key for a fresh submit', async () => {
|
||||
const first = await queue.add('subagent', { prompt: 'keyed' }, { idempotency_key: 'ttl-key-1' }, SUB);
|
||||
await backdate(first.id, 49);
|
||||
await queue.handleWaitingTTL();
|
||||
expect((await queue.getJob(first.id))?.status).toBe('cancelled');
|
||||
const fresh = await queue.add('subagent', { prompt: 'keyed' }, { idempotency_key: 'ttl-key-1' }, SUB);
|
||||
expect(fresh.id).not.toBe(first.id);
|
||||
expect(fresh.status).toBe('waiting');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStats drain/depth extensions', () => {
|
||||
test('by_type carries drained_* split, waiting_now, and oldest_waiting_minutes', async () => {
|
||||
const done = await queue.add('subagent', { prompt: 'done one' }, {}, SUB);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'completed', started_at = now() - interval '2 minutes',
|
||||
finished_at = now() - interval '1 minute' WHERE id = $1`,
|
||||
[done.id],
|
||||
);
|
||||
const ttlVictim = await queue.add('subagent', { prompt: 'shredded' }, {}, SUB);
|
||||
await backdate(ttlVictim.id, 49);
|
||||
await queue.handleWaitingTTL();
|
||||
const waiting = await queue.add('subagent', { prompt: 'still here' }, {}, SUB);
|
||||
await backdate(waiting.id, 3);
|
||||
|
||||
const stats = await queue.getStats();
|
||||
const sub = stats.by_type.find(t => t.name === 'subagent');
|
||||
expect(sub).toBeDefined();
|
||||
expect(sub!.drained_completed).toBe(1);
|
||||
expect(sub!.drained_cancelled).toBe(1);
|
||||
expect(sub!.waiting_now).toBe(1);
|
||||
expect(sub!.oldest_waiting_minutes).toBeGreaterThanOrEqual(170); // ~3h
|
||||
});
|
||||
});
|
||||
|
||||
describe('idempotency_key precedence over param-coalescing', () => {
|
||||
test('a keyed submit never param-coalesces onto an unkeyed identical row', async () => {
|
||||
// Producer-owned idempotency is the STRONGER contract: a param-coalesce
|
||||
// hit would return a row the key was never registered against, so a
|
||||
// later same-key submit would insert fresh and run the work twice.
|
||||
const unkeyed = await queue.add('subagent', { prompt: 'same work' }, {}, SUB);
|
||||
const keyed = await queue.add('subagent', { prompt: 'same work' }, { idempotency_key: 'key-precedence-1' }, SUB);
|
||||
expect(keyed.id).not.toBe(unkeyed.id);
|
||||
expect(keyed.coalesced).not.toBe(true);
|
||||
// And the keyed row carries no param hash — coalescing was disabled, not deferred.
|
||||
expect((keyed.data as Record<string, unknown>).__param_hash).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelJobs reason + rootStatuses (direct)', () => {
|
||||
test('reason sets error_text; no-reason cancel preserves pre-existing error_text', async () => {
|
||||
const a = await queue.add('subagent', { prompt: 'reason target' }, {}, SUB);
|
||||
const b = await queue.add('subagent', { prompt: 'prior-text holder' }, {}, SUB);
|
||||
await engine.executeRaw(`UPDATE minion_jobs SET error_text = 'prior failure detail' WHERE id = $1`, [b.id]);
|
||||
|
||||
await queue.cancelJobs([a.id], { reason: 'explicit cancel reason' });
|
||||
await queue.cancelJobs([b.id]); // no reason → COALESCE keeps what was there
|
||||
|
||||
const aRow = await queue.getJob(a.id);
|
||||
const bRow = await queue.getJob(b.id);
|
||||
expect(aRow?.status).toBe('cancelled');
|
||||
expect(aRow?.error_text).toBe('explicit cancel reason');
|
||||
expect(bRow?.status).toBe('cancelled');
|
||||
expect(bRow?.error_text).toBe('prior failure detail');
|
||||
});
|
||||
|
||||
test('rootStatuses re-check: an active row in the batch is left running', async () => {
|
||||
const w = await queue.add('subagent', { prompt: 'still waiting' }, {}, SUB);
|
||||
const act = await queue.add('subagent', { prompt: 'claimed meanwhile' }, {}, SUB);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'active', lock_until = now() + interval '5 minutes' WHERE id = $1`,
|
||||
[act.id],
|
||||
);
|
||||
const swept = await queue.cancelJobs([w.id, act.id], { reason: 'sweep', rootStatuses: ['waiting'] });
|
||||
expect(swept.some(j => j.id === w.id)).toBe(true);
|
||||
expect(swept.some(j => j.id === act.id)).toBe(false);
|
||||
expect((await queue.getJob(w.id))?.status).toBe('cancelled');
|
||||
expect((await queue.getJob(act.id))?.status).toBe('active');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runWaitingTtlTick warn-before-act gate (D1A)', () => {
|
||||
test('tick 1 = notice (counts affected, stamps ISO flag, cancels NOTHING); grace holds; sweep after grace', async () => {
|
||||
const victim = await queue.add('subagent', { prompt: 'expired long ago' }, {}, SUB);
|
||||
await backdate(victim.id, 49);
|
||||
|
||||
// Tick 1: notice. Nothing cancelled, flag stamped with a timestamp.
|
||||
const t1 = await runWaitingTtlTick(engine, queue);
|
||||
expect(t1.phase).toBe('notice');
|
||||
expect(t1.affected).toBe(1);
|
||||
expect(t1.cancelled).toBe(0);
|
||||
expect((await queue.getJob(victim.id))?.status).toBe('waiting');
|
||||
const flag = await engine.getConfig(TTL_NOTICE_SHOWN_KEY);
|
||||
expect(Number.isFinite(Date.parse(flag ?? ''))).toBe(true);
|
||||
|
||||
// Tick 2 inside the grace window: still nothing cancelled.
|
||||
const t2 = await runWaitingTtlTick(engine, queue);
|
||||
expect(t2.phase).toBe('grace');
|
||||
expect((await queue.getJob(victim.id))?.status).toBe('waiting');
|
||||
|
||||
// Grace elapsed (env seam = 0ms): the sweep runs.
|
||||
await withEnv({ GBRAIN_MINIONS_TTL_NOTICE_GRACE_MS: '0' }, async () => {
|
||||
const t3 = await runWaitingTtlTick(engine, queue);
|
||||
expect(t3.phase).toBe('swept');
|
||||
expect(t3.cancelled).toBe(1);
|
||||
});
|
||||
const after = await queue.getJob(victim.id);
|
||||
expect(after?.status).toBe('cancelled');
|
||||
expect(after?.error_text).toStartWith('waiting_ttl_expired');
|
||||
});
|
||||
|
||||
test("legacy flag value 'true' (pre-grace releases) sweeps immediately", async () => {
|
||||
const victim = await queue.add('subagent', { prompt: 'legacy-flag victim' }, {}, SUB);
|
||||
await backdate(victim.id, 49);
|
||||
await engine.setConfig(TTL_NOTICE_SHOWN_KEY, 'true');
|
||||
const tick = await runWaitingTtlTick(engine, queue);
|
||||
expect(tick.phase).toBe('swept');
|
||||
expect((await queue.getJob(victim.id))?.status).toBe('cancelled');
|
||||
});
|
||||
|
||||
test('kill-switch: GBRAIN_MINIONS_ADMISSION=0 does nothing, not even the notice', async () => {
|
||||
await withEnv({ GBRAIN_MINIONS_ADMISSION: '0' }, async () => {
|
||||
const tick = await runWaitingTtlTick(engine, queue);
|
||||
expect(tick.phase).toBe('killed');
|
||||
});
|
||||
expect(await engine.getConfig(TTL_NOTICE_SHOWN_KEY)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -112,6 +112,70 @@ describe('gbrain quarantine clear', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain quarantine clear — multi-source ambiguity + --source-id', () => {
|
||||
const JUNK = (t: string) => `---\ntitle: ${t}\ntype: note\n---\n\nCloudflare Ray ID: q. junk body.`;
|
||||
|
||||
/** Same slug quarantined in BOTH 'default' and 'work'. */
|
||||
async function seedBothSources(): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, config) VALUES ('work', 'work', '{}'::jsonb) ON CONFLICT (id) DO NOTHING`,
|
||||
);
|
||||
await importFromContent(engine, 'notes/dup', JUNK('D'), { noEmbed: true });
|
||||
await importFromContent(engine, 'notes/dup', JUNK('W'), { noEmbed: true, sourceId: 'work' });
|
||||
for (const src of ['default', 'work']) {
|
||||
const page = await engine.getPage('notes/dup', { sourceId: src });
|
||||
expect(isQuarantined(page!.frontmatter as Record<string, unknown>)).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
test('clear without --source-id on a slug in two sources errors (exit 2) listing the sources; nothing mutated', async () => {
|
||||
await withHome(async () => {
|
||||
await seedBothSources();
|
||||
// The ambiguity path console.errors + process.exit(2)s. Stub exit with a
|
||||
// throw (commands-search.test.ts pattern) so the test can assert on it.
|
||||
const errs: string[] = [];
|
||||
const origErr = console.error;
|
||||
const origExit = process.exit;
|
||||
let exitCode: number | undefined;
|
||||
console.error = (...a: unknown[]) => { errs.push(a.map(String).join(' ')); };
|
||||
(process.exit as unknown as (code?: number) => void) =
|
||||
((code?: number) => { exitCode = code ?? 0; throw new Error('__EXIT__'); }) as never;
|
||||
try {
|
||||
await expect(runQuarantine(engine, ['clear', 'notes/dup'])).rejects.toThrow('__EXIT__');
|
||||
} finally {
|
||||
console.error = origErr;
|
||||
process.exit = origExit;
|
||||
}
|
||||
expect(exitCode).toBe(2);
|
||||
const msg = errs.join('\n');
|
||||
expect(msg).toContain('exists in 2 sources');
|
||||
expect(msg).toContain('default');
|
||||
expect(msg).toContain('work');
|
||||
expect(msg).toContain('--source-id');
|
||||
// Ambiguity is an error, not a coin flip: BOTH rows remain quarantined.
|
||||
for (const src of ['default', 'work']) {
|
||||
const page = await engine.getPage('notes/dup', { sourceId: src });
|
||||
expect(isQuarantined(page!.frontmatter as Record<string, unknown>)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('clear --source-id work clears only the work row; default row untouched', async () => {
|
||||
await withHome(async () => {
|
||||
await seedBothSources();
|
||||
// --force: the stored body is still junk, so an unforced clear would
|
||||
// re-quarantine instead of exercising the deterministic-source write.
|
||||
await capture(() =>
|
||||
runQuarantine(engine, ['clear', 'notes/dup', '--source-id', 'work', '--force', '--no-embed']),
|
||||
);
|
||||
const work = await engine.getPage('notes/dup', { sourceId: 'work' });
|
||||
expect(isQuarantined(work!.frontmatter as Record<string, unknown>)).toBe(false);
|
||||
const def = await engine.getPage('notes/dup', { sourceId: 'default' });
|
||||
expect(isQuarantined(def!.frontmatter as Record<string, unknown>)).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('engine.getContentFlagsByPageIds', () => {
|
||||
test('returns markers for flagged pages, skips clean, empty-input short-circuits', async () => {
|
||||
await withHome(async () => {
|
||||
|
||||
@@ -332,12 +332,14 @@ describe('runAllLintRules — composition', () => {
|
||||
});
|
||||
|
||||
describe('rule registry shape', () => {
|
||||
it('ALL_LINT_RULES contains 12 rules', () => {
|
||||
it('ALL_LINT_RULES contains 14 rules', () => {
|
||||
// v0.41.37.0 #1569 added link_regex_catastrophic_backtrack (file-plane).
|
||||
expect(ALL_LINT_RULES.length).toBe(12);
|
||||
// Five-issue fix wave added stored_type_is_alias + stored_type_undeclared
|
||||
// (both data-plane — they audit the pages corpus against the pack).
|
||||
expect(ALL_LINT_RULES.length).toBe(14);
|
||||
});
|
||||
|
||||
it('FILE_PLANE_LINT_RULES excludes the 2 DB-aware rules', () => {
|
||||
it('FILE_PLANE_LINT_RULES excludes the 4 DB-aware rules', () => {
|
||||
expect(FILE_PLANE_LINT_RULES.length).toBe(10);
|
||||
expect(FILE_PLANE_LINT_RULES.every((r) => !r.planeAware)).toBe(true);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Alias-footgun visibility (issue 3 of the five-issue fix wave).
|
||||
*
|
||||
* gbrain stores explicit frontmatter types literally and never re-normalizes
|
||||
* them, so a type that is an ALIAS of a canonical pack type (or undeclared
|
||||
* entirely) routes silently. These tests pin the mechanism that makes it loud:
|
||||
* classifyStoredType, the import-time type_warning field, the aggregated
|
||||
* sync/import summary renderer, and the data-plane schema lint rules.
|
||||
*
|
||||
* Privacy rule: fixtures use generic type names only.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import {
|
||||
classifyStoredType,
|
||||
sanitizeTypeForDisplay,
|
||||
renderTypeWarningSummary,
|
||||
type TypeUsagePack,
|
||||
} from '../src/core/schema-pack/type-usage.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
|
||||
const PACK: TypeUsagePack = {
|
||||
page_types: [
|
||||
{ name: 'event', path_prefixes: ['events/'], aliases: ['happening', 'occasion'] },
|
||||
{ name: 'note', path_prefixes: ['notes/'], aliases: [] },
|
||||
{ name: 'isolated', path_prefixes: [] },
|
||||
],
|
||||
};
|
||||
|
||||
describe('classifyStoredType', () => {
|
||||
test('canonical page_type name → canonical', () => {
|
||||
expect(classifyStoredType('event', PACK)).toEqual({ kind: 'canonical' });
|
||||
});
|
||||
|
||||
test('alias → alias_of with canonical + filing directory', () => {
|
||||
expect(classifyStoredType('happening', PACK)).toEqual({
|
||||
kind: 'alias_of',
|
||||
canonical: 'event',
|
||||
directory: 'events/',
|
||||
});
|
||||
});
|
||||
|
||||
test('alias of a type with no path_prefixes → directory undefined', () => {
|
||||
const pack: TypeUsagePack = {
|
||||
page_types: [{ name: 'isolated', path_prefixes: [], aliases: ['loner'] }],
|
||||
};
|
||||
const cls = classifyStoredType('loner', pack);
|
||||
expect(cls.kind).toBe('alias_of');
|
||||
expect((cls as { directory?: string }).directory).toBeUndefined();
|
||||
});
|
||||
|
||||
test('undeclared type → undeclared', () => {
|
||||
expect(classifyStoredType('mystery', PACK)).toEqual({ kind: 'undeclared' });
|
||||
});
|
||||
|
||||
test('canonical wins over alias when a name is both (shadowing pack)', () => {
|
||||
const shadowed: TypeUsagePack = {
|
||||
page_types: [
|
||||
{ name: 'event', path_prefixes: ['events/'], aliases: [] },
|
||||
{ name: 'note', path_prefixes: ['notes/'], aliases: ['event'] },
|
||||
],
|
||||
};
|
||||
expect(classifyStoredType('event', shadowed)).toEqual({ kind: 'canonical' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeTypeForDisplay', () => {
|
||||
test('strips control characters (ANSI-escape hygiene)', () => {
|
||||
expect(sanitizeTypeForDisplay('ev\x1b[31mil')).toBe('ev[31mil');
|
||||
expect(sanitizeTypeForDisplay('a\x00b\x07c')).toBe('abc');
|
||||
});
|
||||
|
||||
test('caps length at 64', () => {
|
||||
const long = 'x'.repeat(100);
|
||||
const out = sanitizeTypeForDisplay(long);
|
||||
expect(out.length).toBe(64);
|
||||
expect(out.endsWith('...')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderTypeWarningSummary', () => {
|
||||
test('one line per distinct type, alias line names canonical + directory', () => {
|
||||
const lines = renderTypeWarningSummary([
|
||||
{ kind: 'alias_of', type: 'happening', canonical: 'event', directory: 'events/', count: 12 },
|
||||
{ kind: 'undeclared', type: 'mystery', count: 3 },
|
||||
]);
|
||||
expect(lines).toHaveLength(2);
|
||||
expect(lines[0]).toContain("'happening'");
|
||||
expect(lines[0]).toContain("'event'");
|
||||
expect(lines[0]).toContain('events/');
|
||||
expect(lines[0]).toContain('12 file(s)');
|
||||
expect(lines[1]).toContain("'mystery'");
|
||||
expect(lines[1]).toContain('not declared');
|
||||
});
|
||||
|
||||
test('empty input → no lines', () => {
|
||||
expect(renderTypeWarningSummary([])).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('importFromContent type_warning (advisory, type stored literally)', () => {
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
async function importWithType(slug: string, type: string | null, activePack?: TypeUsagePack) {
|
||||
const { importFromContent } = await import('../src/core/import-file.ts');
|
||||
const fm = type === null ? '' : `type: ${type}\n`;
|
||||
const md = `---\n${fm}title: T\n---\n\n# T\n\nBody ${Math.random().toString(36).slice(2)}.\n`;
|
||||
return importFromContent(engine, slug, md, {
|
||||
noEmbed: true,
|
||||
...(activePack ? { activePack: activePack as never } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
test('explicit alias type → alias_of warning, type stored as-is', async () => {
|
||||
const r = await importWithType('notes/alias-page', 'happening', PACK);
|
||||
expect(r.status).toBe('imported');
|
||||
expect(r.type_warning).toEqual({
|
||||
kind: 'alias_of',
|
||||
type: 'happening',
|
||||
canonical: 'event',
|
||||
directory: 'events/',
|
||||
});
|
||||
const page = await engine.getPage('notes/alias-page');
|
||||
expect(page?.type).toBe('happening'); // stored literally — advisory only
|
||||
}, 30_000);
|
||||
|
||||
test('explicit undeclared type → undeclared warning', async () => {
|
||||
const r = await importWithType('notes/mystery-page', 'mystery', PACK);
|
||||
expect(r.type_warning).toEqual({ kind: 'undeclared', type: 'mystery' });
|
||||
}, 30_000);
|
||||
|
||||
test('explicit canonical type → no warning', async () => {
|
||||
const r = await importWithType('notes/canonical-page', 'event', PACK);
|
||||
expect(r.type_warning).toBeUndefined();
|
||||
}, 30_000);
|
||||
|
||||
test('no activePack → no warning (classification skipped)', async () => {
|
||||
const r = await importWithType('notes/packless-page', 'happening');
|
||||
expect(r.type_warning).toBeUndefined();
|
||||
}, 30_000);
|
||||
|
||||
test('no explicit type (typeExplicit false) → no warning', async () => {
|
||||
const r = await importWithType('notes/implicit-page', null, PACK);
|
||||
expect(r.type_warning).toBeUndefined();
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('schema lint data-plane rules: stored_type_is_alias / stored_type_undeclared', () => {
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
await engine.putPage('events/party', {
|
||||
type: 'happening', title: 'P', compiled_truth: 'x', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('notes/odd', {
|
||||
type: 'mystery', title: 'M', compiled_truth: 'x', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('events/legit', {
|
||||
type: 'event', title: 'E', compiled_truth: 'x', timeline: '', frontmatter: {},
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
const manifest = {
|
||||
name: 'test-pack',
|
||||
page_types: [
|
||||
{ name: 'event', path_prefixes: ['events/'], aliases: ['happening'] },
|
||||
{ name: 'note', path_prefixes: ['notes/'], aliases: [] },
|
||||
],
|
||||
} as never;
|
||||
|
||||
test('alias-typed corpus rows surface as stored_type_is_alias warnings', async () => {
|
||||
const { storedTypeIsAlias } = await import('../src/core/schema-pack/lint-rules.ts');
|
||||
const issues = await storedTypeIsAlias(manifest, { engine });
|
||||
const hit = issues.find(i => i.type === 'happening');
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit!.severity).toBe('warning');
|
||||
expect(hit!.message).toContain("alias of 'event'");
|
||||
}, 30_000);
|
||||
|
||||
test('undeclared-typed corpus rows surface as stored_type_undeclared warnings', async () => {
|
||||
const { storedTypeUndeclared } = await import('../src/core/schema-pack/lint-rules.ts');
|
||||
const issues = await storedTypeUndeclared(manifest, { engine });
|
||||
const hit = issues.find(i => i.type === 'mystery');
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit!.severity).toBe('warning');
|
||||
// canonical + alias types must NOT be flagged
|
||||
expect(issues.find(i => i.type === 'event')).toBeUndefined();
|
||||
expect(issues.find(i => i.type === 'happening')).toBeUndefined();
|
||||
}, 30_000);
|
||||
|
||||
test('rules are engine-gated: no engine → no issues', async () => {
|
||||
const { storedTypeIsAlias, storedTypeUndeclared } = await import('../src/core/schema-pack/lint-rules.ts');
|
||||
expect(await storedTypeIsAlias(manifest, {})).toHaveLength(0);
|
||||
expect(await storedTypeUndeclared(manifest, {})).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -122,3 +122,118 @@ describe('source-id routing — FK integrity (v0.36.x #1078)', () => {
|
||||
).resolves.toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unscoped-check/scoped-write pairs (five-issue fix wave, commit 4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('unscoped-check/scoped-write regressions (writer + slug-registry + import-file)', () => {
|
||||
test('REGRESSION-CRITICAL: importFromContent without sourceId reads the DEFAULT row, not any-source', async () => {
|
||||
// Pre-fix: the existence check matched people/carol in 'work' while the
|
||||
// write targeted 'default' — read-A/write-B divergence. Post-fix the read
|
||||
// mirrors the write's schema default.
|
||||
await importFromContent(engine, 'people/carol', '---\ntype: person\ntitle: Work Carol\n---\nWork row.', {
|
||||
noEmbed: true,
|
||||
sourceId: 'work',
|
||||
});
|
||||
const workBefore = await engine.executeRaw<{ content_hash: string }>(
|
||||
`SELECT content_hash FROM pages WHERE slug = 'people/carol' AND source_id = 'work'`,
|
||||
);
|
||||
|
||||
await importFromContent(engine, 'people/carol', '---\ntype: person\ntitle: Default Carol\n---\nDefault row.', {
|
||||
noEmbed: true,
|
||||
});
|
||||
|
||||
const rows = await engine.executeRaw<{ source_id: string; title: string; content_hash: string }>(
|
||||
`SELECT source_id, title, content_hash FROM pages WHERE slug = 'people/carol' ORDER BY source_id`,
|
||||
);
|
||||
// Two independent rows: the no-sourceId import created/updated 'default'
|
||||
// and left 'work' byte-identical.
|
||||
expect(rows.length).toBe(2);
|
||||
const work = rows.find(r => r.source_id === 'work')!;
|
||||
const def = rows.find(r => r.source_id === 'default')!;
|
||||
expect(work.title).toBe('Work Carol');
|
||||
expect(work.content_hash).toBe(workBefore[0].content_hash);
|
||||
expect(def.title).toBe('Default Carol');
|
||||
});
|
||||
|
||||
test('scoped BrainWriter creates the exact slug in its source even when the slug is taken elsewhere', async () => {
|
||||
const { BrainWriter } = await import('../src/core/output/writer.ts');
|
||||
// Slug exists ONLY in 'personal'.
|
||||
await importFromContent(engine, 'people/dave', '---\ntype: person\ntitle: Personal Dave\n---\nx', {
|
||||
noEmbed: true,
|
||||
sourceId: 'personal',
|
||||
});
|
||||
|
||||
const writer = new BrainWriter(engine, { strictMode: 'off', sourceId: 'work' });
|
||||
const ctx = {
|
||||
engine, config: {}, requestId: 't', remote: false as const,
|
||||
logger: { info: () => {}, warn: () => {}, error: () => {} },
|
||||
};
|
||||
const { result: slug } = await writer.transaction(async (tx) => {
|
||||
return tx.createEntity({
|
||||
desiredSlug: 'people/dave',
|
||||
displayName: 'Work Dave',
|
||||
type: 'person',
|
||||
compiledTruth: 'Work dave body.',
|
||||
});
|
||||
}, ctx as never);
|
||||
|
||||
// Pre-fix: the unscoped probe saw personal's row and disambiguated to
|
||||
// people/dave-2. Post-fix: exact slug, in the writer's source.
|
||||
expect(slug).toBe('people/dave');
|
||||
const rows = await engine.executeRaw<{ source_id: string; title: string }>(
|
||||
`SELECT source_id, title FROM pages WHERE slug = 'people/dave' ORDER BY source_id`,
|
||||
);
|
||||
expect(rows.length).toBe(2);
|
||||
expect(rows.find(r => r.source_id === 'work')?.title).toBe('Work Dave');
|
||||
expect(rows.find(r => r.source_id === 'personal')?.title).toBe('Personal Dave');
|
||||
});
|
||||
|
||||
test('scoped writer setFrontmatterField edits ITS row and never the other source', async () => {
|
||||
const { BrainWriter } = await import('../src/core/output/writer.ts');
|
||||
await importFromContent(engine, 'people/erin', '---\ntype: person\ntitle: Work Erin\n---\nx', {
|
||||
noEmbed: true, sourceId: 'work',
|
||||
});
|
||||
await importFromContent(engine, 'people/erin', '---\ntype: person\ntitle: Personal Erin\n---\ny', {
|
||||
noEmbed: true, sourceId: 'personal',
|
||||
});
|
||||
|
||||
const writer = new BrainWriter(engine, { strictMode: 'off', sourceId: 'work' });
|
||||
const ctx = {
|
||||
engine, config: {}, requestId: 't', remote: false as const,
|
||||
logger: { info: () => {}, warn: () => {}, error: () => {} },
|
||||
};
|
||||
await writer.transaction(async (tx) => {
|
||||
await tx.setFrontmatterField('people/erin', 'reviewed', true);
|
||||
}, ctx as never);
|
||||
|
||||
const work = await engine.getPage('people/erin', { sourceId: 'work' });
|
||||
const personal = await engine.getPage('people/erin', { sourceId: 'personal' });
|
||||
expect(work?.frontmatter?.reviewed).toBe(true);
|
||||
expect(personal?.frontmatter?.reviewed).toBeUndefined();
|
||||
});
|
||||
|
||||
test('unscoped getPage on a duplicated slug is deterministic: default-source-first', async () => {
|
||||
// 'archive' sorts before 'default' — pre-fix (no ORDER BY) the winner was
|
||||
// arbitrary; plain alpha would wrongly prefer 'archive'.
|
||||
await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ('archive', 'archive') ON CONFLICT DO NOTHING`);
|
||||
await importFromContent(engine, 'people/frank', '---\ntype: person\ntitle: Archive Frank\n---\nx', {
|
||||
noEmbed: true, sourceId: 'archive',
|
||||
});
|
||||
await importFromContent(engine, 'people/frank', '---\ntype: person\ntitle: Default Frank\n---\ny', {
|
||||
noEmbed: true,
|
||||
});
|
||||
|
||||
const page = await engine.getPage('people/frank'); // gbrain-allow-unscoped-getpage: pinning the deterministic default-first tiebreak itself
|
||||
expect(page?.title).toBe('Default Frank');
|
||||
|
||||
// And when only non-default sources hold the slug, the tiebreak is stable alpha.
|
||||
await engine.deletePage('people/frank', { sourceId: 'default' });
|
||||
await importFromContent(engine, 'people/frank', '---\ntype: person\ntitle: Work Frank\n---\nz', {
|
||||
noEmbed: true, sourceId: 'work',
|
||||
});
|
||||
const page2 = await engine.getPage('people/frank'); // gbrain-allow-unscoped-getpage: pinning the deterministic alpha tiebreak itself
|
||||
expect(page2?.title).toBe('Archive Frank');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,7 +11,9 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
isSyncable,
|
||||
isPoisonedPath,
|
||||
unsyncableReason,
|
||||
sanitizePathForDisplay,
|
||||
SYNC_SKIP_FILES,
|
||||
type SyncableReason,
|
||||
} from '../src/core/sync.ts';
|
||||
@@ -32,6 +34,13 @@ describe('#1433 — isSyncable / unsyncableReason are duals of one classifier',
|
||||
{ path: 'vendor/pkg/note.md', expected: 'pruned-dir', note: 'vendor/ is pruned' },
|
||||
{ path: '.git/notes.md', expected: 'pruned-dir', note: 'hidden dir pruned' },
|
||||
{ path: 'node_modules/foo/README.md', expected: 'pruned-dir', note: 'node_modules pruned' },
|
||||
// Poisoned-path incident class: markdown-link syntax as a literal filename.
|
||||
{ path: '[atoms/foo.md](https:/example).md', expected: 'malformed-path', note: 'markdown-link-shaped junk filename rejected' },
|
||||
{ path: 'notes/[wip] draft.md', expected: 'malformed-path', note: 'bare bracket in MARKDOWN filename rejected' },
|
||||
{ path: 'notes/bell' + '\x07' + '.md', expected: 'malformed-path', note: 'control character in filename rejected' },
|
||||
{ path: 'docs/[locale]/guide.md', expected: 'malformed-path', note: 'bracket DIRECTORY segment above a markdown file rejected' },
|
||||
{ path: 'notes/meeting (1).md', expected: null, note: 'parens are legitimate filename characters — deliberately allowed' },
|
||||
{ path: 'people/alice.txt](x', expected: 'strategy', note: 'strategy check wins before malformed-path (classifier ordering keeps reconcile strategy-safe)' },
|
||||
];
|
||||
|
||||
for (const c of cases) {
|
||||
@@ -62,4 +71,35 @@ describe('#1433 — isSyncable / unsyncableReason are duals of one classifier',
|
||||
expect(isSyncable(c.path)).toBe(unsyncableReason(c.path) === null);
|
||||
}
|
||||
});
|
||||
|
||||
test('brackets are markdown-scoped: code-strategy framework paths stay syncable', () => {
|
||||
// Cross-model adversarial finding: a blanket bracket rejection would have
|
||||
// rejected ubiquitous Next.js/Nuxt dynamic-route layouts in code lanes
|
||||
// AND reconcile-deleted their previously indexed rows.
|
||||
expect(unsyncableReason('app/[id]/page.tsx', { strategy: 'code' })).toBeNull();
|
||||
expect(unsyncableReason('app/[...slug]/route.ts', { strategy: 'code' })).toBeNull();
|
||||
// Same path under the default markdown strategy is a STRATEGY rejection
|
||||
// (ordering: strategy classifies first), never malformed-path.
|
||||
expect(unsyncableReason('app/[id]/page.tsx')).toBe('strategy');
|
||||
// A markdown file under a bracket dir is malformed even in code/auto lanes.
|
||||
expect(unsyncableReason('app/[id]/README.md', { strategy: 'auto' })).toBe('malformed-path');
|
||||
});
|
||||
|
||||
test('isPoisonedPath: only the injection signature is sweepable', () => {
|
||||
expect(isPoisonedPath('[atoms/foo.md](https:/example).md')).toBe(true); // `](`
|
||||
expect(isPoisonedPath('notes/bell' + '\x07' + '.md')).toBe(true); // control char
|
||||
expect(isPoisonedPath('notes [draft].md')).toBe(false); // bare brackets: rows survive
|
||||
expect(isPoisonedPath('docs/[locale]/guide.md')).toBe(false);
|
||||
});
|
||||
|
||||
test('sanitizePathForDisplay: control bytes → U+FFFD, DEL included, brackets kept, 200-char cap', () => {
|
||||
expect(sanitizePathForDisplay('notes/bell' + '\x07' + '.md')).toBe('notes/bell�.md');
|
||||
expect(sanitizePathForDisplay('esc' + '\x1b' + '[31mred')).toBe('esc�[31mred'); // ANSI CSI neutered
|
||||
expect(sanitizePathForDisplay('del' + '\x7f' + '.md')).toBe('del�.md');
|
||||
expect(sanitizePathForDisplay('[foo.md](https-example).md')).toBe('[foo.md](https-example).md');
|
||||
const long = 'a'.repeat(250);
|
||||
const capped = sanitizePathForDisplay(long);
|
||||
expect(capped.length).toBe(200);
|
||||
expect(capped.endsWith('...')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* Malformed-path rejection (poisoned-filename incident regression).
|
||||
*
|
||||
* Files literally named like markdown links (`[atoms/foo.md](https:/...)`)
|
||||
* were ingested without complaint — slugifySegment STRIPS brackets instead of
|
||||
* rejecting them, so junk files minted plausible slugs and polluted search.
|
||||
*
|
||||
* The fix adds SyncableReason 'malformed-path' to the classifier. These tests
|
||||
* pin the OPERATIONAL consequences, which are subtle (two prior outside-voice
|
||||
* findings live here):
|
||||
*
|
||||
* 1. REGRESSION-CRITICAL: a DELETE of a malformed path must still process —
|
||||
* naively filtering manifest.deleted by isSyncable would orphan the
|
||||
* previously-ingested DB row forever (searchable junk with no way out).
|
||||
* 2. A MODIFIED malformed file sweeps its stale DB row (the cleanup lane
|
||||
* treats 'malformed-path' as delete-eligible, unlike metafile/pruned-dir).
|
||||
* 3. The full-sync reconcile treats malformed-path rows as eligible, so a
|
||||
* full sync removes previously-ingested poison EVEN IF the junk file is
|
||||
* still sitting in the repo.
|
||||
* 4. importFromFile's defense-in-depth returns an INFORMATIONAL skip
|
||||
* (skip_reason='malformed_path') that never lands in failedFiles — a
|
||||
* junk filename must not gate bookmark advancement.
|
||||
*
|
||||
* Marked .serial.test.ts: spawns git subprocesses and shares one PGLite
|
||||
* engine across tests.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let repoPath: string;
|
||||
|
||||
const JUNK_NAME = '[foo.md](https-example).md';
|
||||
|
||||
function gitInit(repo: string): void {
|
||||
execSync('git init', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git config user.email "test@test.com"', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git config user.name "Test"', { cwd: repo, stdio: 'pipe' });
|
||||
}
|
||||
|
||||
/** Seed a page row as if a pre-fix gbrain had ingested the junk file. */
|
||||
async function seedPoisonedRow(slug: string, sourcePath: string): Promise<void> {
|
||||
await engine.putPage(slug, {
|
||||
type: 'note',
|
||||
title: 'Poisoned page',
|
||||
compiled_truth: 'Junk row ingested before the malformed-path gate existed.',
|
||||
timeline: '',
|
||||
frontmatter: { type: 'note' },
|
||||
});
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET source_path = $1 WHERE slug = $2`,
|
||||
[sourcePath, slug],
|
||||
);
|
||||
}
|
||||
|
||||
describe('malformed-path sync semantics (poisoned-filename incident)', () => {
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-malformed-'));
|
||||
gitInit(repoPath);
|
||||
mkdirSync(join(repoPath, 'topics'), { recursive: true });
|
||||
writeFileSync(join(repoPath, 'topics/foo.md'), [
|
||||
'---',
|
||||
'type: concept',
|
||||
'title: Foo',
|
||||
'---',
|
||||
'',
|
||||
'Baseline content.',
|
||||
].join('\n'));
|
||||
execSync('git add -A && git commit -m "initial"', { cwd: repoPath, stdio: 'pipe' });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('sync never imports a bracket-named file, and the skip is not a failure', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
writeFileSync(join(repoPath, JUNK_NAME), '# junk\n');
|
||||
execSync('git add -A && git commit -m "junk lands"', { cwd: repoPath, stdio: 'pipe' });
|
||||
|
||||
const result = await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
|
||||
// The junk file must not block the sync or land in the index.
|
||||
expect(['first_sync', 'synced', 'up_to_date']).toContain(result.status);
|
||||
const rows = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE source_path = $1`, [JUNK_NAME],
|
||||
);
|
||||
expect(rows).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
test('REGRESSION-CRITICAL: deleting a junk file still deletes its pre-existing DB row', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
writeFileSync(join(repoPath, JUNK_NAME), '# junk\n');
|
||||
execSync('git add -A && git commit -m "junk lands"', { cwd: repoPath, stdio: 'pipe' });
|
||||
const first = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
expect(first.status).not.toBe('blocked_by_failures');
|
||||
|
||||
// Simulate the pre-fix world: the junk row is already in the DB.
|
||||
await seedPoisonedRow('atoms/foo-md-https-example', JUNK_NAME);
|
||||
|
||||
execSync(`git rm '${JUNK_NAME}'`, { cwd: repoPath, stdio: 'pipe' });
|
||||
execSync('git commit -m "remove junk"', { cwd: repoPath, stdio: 'pipe' });
|
||||
|
||||
const second = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
expect(second.status).not.toBe('blocked_by_failures');
|
||||
|
||||
// Without the delete-lane carve-out the row would be orphaned forever.
|
||||
const survivor = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE source_path = $1 AND deleted_at IS NULL`, [JUNK_NAME],
|
||||
);
|
||||
expect(survivor).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
test('a MODIFIED junk file sweeps its stale DB row (cleanup lane treats malformed-path as delete-eligible)', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
writeFileSync(join(repoPath, JUNK_NAME), '# junk\n');
|
||||
execSync('git add -A && git commit -m "junk lands"', { cwd: repoPath, stdio: 'pipe' });
|
||||
await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
|
||||
await seedPoisonedRow('atoms/foo-md-https-example', JUNK_NAME);
|
||||
|
||||
writeFileSync(join(repoPath, JUNK_NAME), '# junk edited\n');
|
||||
execSync('git add -A && git commit -m "edit junk"', { cwd: repoPath, stdio: 'pipe' });
|
||||
await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
|
||||
const survivor = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE source_path = $1 AND deleted_at IS NULL`, [JUNK_NAME],
|
||||
);
|
||||
expect(survivor).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
test('full-sync reconcile sweeps poisoned rows even while the junk file still exists on disk', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
// Junk file committed AND still present in the working tree.
|
||||
writeFileSync(join(repoPath, JUNK_NAME), '# junk\n');
|
||||
execSync('git add -A && git commit -m "junk lands"', { cwd: repoPath, stdio: 'pipe' });
|
||||
|
||||
// Poisoned row exists in the default source (pre-fix ingestion).
|
||||
await seedPoisonedRow('atoms/foo-md-https-example', JUNK_NAME);
|
||||
|
||||
// A sourceId-scoped FULL sync runs the delete-reconcile pass.
|
||||
const result = await performSync(engine, {
|
||||
repoPath, full: true, noPull: true, noEmbed: true, sourceId: 'default',
|
||||
});
|
||||
expect(result.status).not.toBe('blocked_by_failures');
|
||||
|
||||
const survivor = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE source_path = $1 AND deleted_at IS NULL`, [JUNK_NAME],
|
||||
);
|
||||
expect(survivor).toHaveLength(0);
|
||||
|
||||
// Sanity: the healthy page was not collateral damage.
|
||||
const healthy = await engine.getPage('topics/foo');
|
||||
expect(healthy).not.toBeNull();
|
||||
}, 60_000);
|
||||
|
||||
test('bare-bracket markdown row SURVIVES reconcile (only the poison signature is sweepable)', async () => {
|
||||
// Cross-model adversarial finding: a pre-gate release imported
|
||||
// `notes [draft].md` fine (brackets were slug-stripped). The file still
|
||||
// exists on disk; hard-deleting its row on a routine post-upgrade full
|
||||
// sync would be silent data loss. Only `](`/control-char paths sweep.
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const BARE = 'notes [draft].md';
|
||||
|
||||
writeFileSync(join(repoPath, BARE), '# legit draft\n');
|
||||
execSync('git add -A && git commit -m "bare-bracket note"', { cwd: repoPath, stdio: 'pipe' });
|
||||
await seedPoisonedRow('notes-draft', BARE);
|
||||
|
||||
const result = await performSync(engine, {
|
||||
repoPath, full: true, noPull: true, noEmbed: true, sourceId: 'default',
|
||||
});
|
||||
expect(result.status).not.toBe('blocked_by_failures');
|
||||
|
||||
const survivor = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE source_path = $1 AND deleted_at IS NULL`, [BARE],
|
||||
);
|
||||
expect(survivor).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
test('importFromFile defense: bracket filename → informational skip with skip_reason, brackets never stripped into a slug', async () => {
|
||||
const { importFromFile } = await import('../src/core/import-file.ts');
|
||||
const abs = join(repoPath, JUNK_NAME);
|
||||
writeFileSync(abs, '# junk\n');
|
||||
const result = await importFromFile(engine, abs, JUNK_NAME, { noEmbed: true });
|
||||
expect(result.status).toBe('skipped');
|
||||
expect(result.skip_reason).toBe('malformed_path');
|
||||
expect(result.error).toContain('Rename the file');
|
||||
// The old behavior minted a bracket-stripped slug; the defense returns none.
|
||||
expect(result.slug).toBe('');
|
||||
}, 60_000);
|
||||
});
|
||||
Reference in New Issue
Block a user