mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-15 09:22:18 +00:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19af788774 | ||
|
|
0db03e9c01 | ||
|
|
0c84367e5e | ||
|
|
8a2c233d94 | ||
|
|
d644f18b22 | ||
|
|
5df46f8215 | ||
|
|
56f80310be | ||
|
|
b425c411d5 | ||
|
|
d7175253b3 | ||
|
|
fe3b39d944 | ||
|
|
e37e89ad03 | ||
|
|
21c3f4ced8 | ||
|
|
53f49c5f2c | ||
|
|
958cb2b3f7 | ||
|
|
f10af869f4 | ||
|
|
0a073a64d1 | ||
|
|
65d6babc0e | ||
|
|
141a8ad1b9 | ||
|
|
7e646f577a | ||
|
|
7a424745d9 | ||
|
|
f8ff610b46 |
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-runbook-stamp: 0.45.15.0 -->
|
||||
<!-- gbrain-runbook-stamp: 0.45.16.0 -->
|
||||
<!-- This stamp must equal the VERSION file at every release; CI enforces it
|
||||
(scripts/check-bootstrap-tag.sh). `gbrain bootstrap status` compares it to
|
||||
the installed binary and warns on skew. -->
|
||||
|
||||
@@ -2,6 +2,26 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.45.16.0] - 2026-08-14
|
||||
|
||||
**Fix wave W0: the verified-bug hotfix pass of the code-smell series.** A 10-auditor sweep of the codebase produced 122 findings; the top claims were adversarially verified, and this release fixes every verified live bug — the ones that survived the skeptic pass. Long-running brains get the biggest wins: background cycles can no longer silently run twice, dead background jobs no longer strand their parents, and image search no longer silently degrades after re-embedding. Developers get a test suite that runs 10x faster.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **The background-cycle lock is now actually refreshed while a cycle runs.** Long phases (synthesis, pattern extraction, consolidation — up to 35-minute waits) previously outlived the 5-minute lock TTL with no heartbeat, so a second cycle could start against the same brain and both would write concurrently — duplicated LLM spend and racy writes on Postgres/Supabase brains. A dedicated refresher now heartbeats the lock, every refresh and release is fenced to the exact acquisition (a recycled PID or a superseded run can never touch a successor's lock — including the PGLite file lock, which is no longer rewritten after a detected steal), and a run that loses its lock stops at the next phase boundary with a structured `lock_stolen` report instead of compounding. The job supervisor treats a fenced miss as certain loss and exits for a clean restart.
|
||||
- **Background jobs that die from repeated stalls now notify and unblock their waiting parents.** Previously an aggregator parent whose child was dead-lettered by the stall sweep waited forever; a self-healing sweep also releases parents stranded before the upgrade.
|
||||
- **Retried jobs no longer burn their wall-clock budget while waiting in backoff.** Every automatic re-run path — and every parent-unblock path — resets the per-attempt clock, so exponential-backoff retries and long-waiting aggregators aren't dead-lettered before executing a line.
|
||||
- **Re-embedding no longer flips image chunks to text.** `gbrain embed --stale` (including the autopilot path) preserved every chunk field except `modality`, silently zeroing image retrieval until the next full import. One shared carry list now serves every re-embed path.
|
||||
- **A failed first sync no longer kills the MCP server.** Import preflight failures (missing embedding credentials, unreadable target) now surface as normal tool errors instead of terminating the serving process mid-call.
|
||||
- **`gbrain lint --fix` reports the true fix count** (it previously scanned everything twice and reported "0 auto-fixed" after fixing issues) and walks the tree once.
|
||||
- **The PGLite repair and re-init confirmation prompts can no longer hang forever** on closed or piped stdin: EOF declines safely, and prompts write to stderr so `--json` output stays clean.
|
||||
|
||||
### Changed
|
||||
|
||||
- **`bun run test` is ~10x faster** (measured: a full parallel suite run dropped from ~82 to ~8 minutes). The PGLite schema snapshot is now default-on for the everyday test loop, rebuilt automatically when migrations or the pinned embedding shape change, concurrency-safe across parallel shards and workspaces, and refused on any shape mismatch so a wrong fixture can never poison the suite.
|
||||
- **CI guards now prove they can fail.** A guard registry classifies all 45 check scripts; self-tested scanner guards run against known-bad fixtures on every verify (the registry tracks fixture coverage for the rest), so a guard whose pattern rots into a permanently-green no-op fails the build instead of masquerading as coverage. Two such rotted patterns were found and fixed in the process, along with three guards that were wired into a registry nobody ran.
|
||||
|
||||
To take advantage of v0.45.16.0: upgrade and restart any long-running `gbrain serve`, autopilot, or jobs supervisor/worker daemon so the fenced lock refresh and job-reaper fixes take effect. If you run image search, run `gbrain backfill modality` once after upgrading to restore any image chunks a prior re-embed flipped to text (`gbrain doctor` surfaces the affected count and the exact command). No schema migration and no config changes are required.
|
||||
## [0.45.15.0] - 2026-08-14
|
||||
|
||||
**The queue that drains itself: three background-jobs fixes reported from a downstream agent deployment (upstream issues #2, #3, #4).** A brain whose autopilot cycle stalled mid-run could accumulate byte-identical queued cycles forever while every long job queued before an upgrade died minutes in — and the operator diagnosing it couldn't even find the worker entry point, because `gbrain jobs --help` printed a one-line stub. All three failure modes are closed, and the queue now tells you when it's holding work back.
|
||||
|
||||
@@ -70,7 +70,11 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
imports use static top-level imports. Besides the snapshot loader's lazy
|
||||
`require()` cluster in `pglite-engine.ts:tryLoadSnapshot` (fs/crypto/
|
||||
migrate/pglite-schema + one gateway shape lookup — lazy so production
|
||||
builds without the test-fixture path don't eager-load; the guard now
|
||||
matches `require()` calls too), the only dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
|
||||
+12
-6
@@ -92,11 +92,11 @@ The canonical reference for test tiers, isolation rules, timing, and the E2E
|
||||
lifecycle is [`docs/TESTING.md`](docs/TESTING.md). The short version:
|
||||
|
||||
```bash
|
||||
# Inner edit loop (~85s on a Mac dev box)
|
||||
bun run test # parallel 4-shard fan-out (memory-adaptive) + serial post-pass
|
||||
# Inner edit loop (~8min full suite on a Mac dev box; single files in seconds)
|
||||
bun run test # parallel 4-shard fan-out (memory-adaptive) + serial post-pass; PGLite snapshot default-on
|
||||
bun test test/markdown.test.ts # specific unit test
|
||||
|
||||
# Pre-push gate (19+ parallel checks + typecheck)
|
||||
# Pre-push gate (40+ parallel checks + typecheck)
|
||||
bun run verify
|
||||
|
||||
# Pre-merge sanity (everything CI runs)
|
||||
@@ -115,7 +115,7 @@ DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run t
|
||||
DATABASE_URL=postgresql://... bun run test:e2e
|
||||
```
|
||||
|
||||
Use `bun run verify` before pushing. It runs 19+ guard checks in parallel
|
||||
Use `bun run verify` before pushing. It runs 40+ guard checks in parallel
|
||||
(`scripts/run-verify-parallel.sh`), including: banned fork-name leaks
|
||||
(`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation
|
||||
patterns (`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
@@ -124,8 +124,14 @@ patterns (`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
loop" below), silent fallback to recursive chunking in the compiled binary
|
||||
(`scripts/check-wasm-embedded.sh`), stale admin-dashboard build artifacts
|
||||
(`scripts/check-admin-build.sh`), resolver drift on bundled skills
|
||||
(`bun run check:resolver`), and typecheck. `bun run check:all` runs the full
|
||||
historical sweep including the trailing-newline and exports-count checks.
|
||||
(`bun run check:resolver`), and typecheck. The guard REGISTRY is
|
||||
`scripts/guards-manifest.tsv`, and `scripts/guard-self-test.sh` (also in
|
||||
`verify`) proves each self-tested scanner guard (`selftest=yes` in the
|
||||
manifest; coverage ratchets up from the `todo` rows) can actually fail by
|
||||
running it against known-bad fixtures — a new `scripts/check-*` guard must be
|
||||
registered in the manifest or the build fails. There is no `check:all` script; the
|
||||
trailing-newline, exports-count, and no-legacy-getconnection checks run in
|
||||
`verify` with everything else.
|
||||
|
||||
### Writing tests that survive the parallel loop
|
||||
|
||||
|
||||
@@ -1,5 +1,34 @@
|
||||
# TODOS
|
||||
|
||||
## Code-smell fix-wave deferrals (filed at W0; plan: ~/.claude/plans/system-instruction-you-are-working-encapsulated-eclipse.md)
|
||||
|
||||
Each was individually decided as a deferral in the CEO/eng reviews of the
|
||||
fix-wave plan; the wave series (W0.5–W9, 3.4, 3.6) tracks its own scope there.
|
||||
|
||||
- [ ] **Full engine staged merge** (~10 domains onto shared query modules +
|
||||
Dialect record). **Priority: P2.** Gated on the W9 two-slice pilot criteria
|
||||
(structure+params+results parity on chronicle AND the searchKeyword/CJK
|
||||
hard seam; ≥40% domain LOC cut; Dialect ≤~6 fields; query-builder extension
|
||||
≤~150 lines). The terminal fix for the engine-divergence/JSONB class —
|
||||
blast radius is the production hot path, hence pilot-gated. Blocked by: W9.
|
||||
- [ ] **gateway.ts file split** behind a re-export facade (~121 import sites
|
||||
unmoved). **Priority: P3.** After W8's behavior changes so the split is
|
||||
pure motion; needs the CLAUDE.md engine-dynamic-import exemption-path
|
||||
chasers + check-engine-dynamic-import.sh + build:llms.
|
||||
- [ ] **BrainEngine 149-method interface → domain repos** (65 methods have
|
||||
0-1 callers; 3 already deleted in W3). **Priority: P3.** Shape informed by
|
||||
the W9 pilot's query-module seam.
|
||||
- [ ] **Legacy Anthropic-SDK subagent loop deletion.** **Priority: P2.** One
|
||||
release after W8 flips `agent.use_gateway_loop` default ON (flag stays as
|
||||
the revert path for that release).
|
||||
- [ ] **Deeper test-suite speedup** beyond the W0 snapshot default-on (which
|
||||
already cut the full parallel suite ~4,900s → ~490s). **Priority: P3.**
|
||||
Revisit with post-W0 timing data; diminishing returns until measured.
|
||||
- [ ] **PGLite schema build-time derivation** from SCHEMA_SQL via a named
|
||||
transform list. **Priority: P3.** Only if W3's schema drift TEST proves
|
||||
annoying in practice — the test alone kills the drift bug class (Codex
|
||||
D4.8/D5.23: fresh-schema equivalence ≠ upgrade correctness; old-shape
|
||||
bootstrap fixtures + replay coverage stay regardless).
|
||||
## Jobs fix-wave follow-ups (filed v0.45.15.0 — upstream issues #2/#3/#4)
|
||||
|
||||
- [ ] **P2 — `jobs submit --max-pending` public flag.** maxPending stays an
|
||||
@@ -1840,18 +1869,14 @@ single canonical `src/core/model-pricing.ts` with `canonicalLookup`.
|
||||
operator pipes directly into `crontab -e` instead of copy-paste-massage.
|
||||
~80 LOC. Mirrors `gbrain sync --break-lock` argv shape.
|
||||
|
||||
- **TODO-OPS-2 (P2)**: Lock-loss detection — extend `DbLockHandle.refresh()`
|
||||
to throw `LockLostError` on 0 rows affected. Codex caught during the
|
||||
v0.41.19.0 plan review: `refresh()` runs `UPDATE ... WHERE holder_pid = pid`
|
||||
with no rows-affected check (`db-lock.ts:108-114`, `:151-156`). If the
|
||||
TTL expired and another worker took over, the original keeps writing
|
||||
silently. v0.41.19.0 ships TTL=5min + active in-phase refresh via
|
||||
`buildYieldDuringPhase` which makes the race window much narrower, but
|
||||
an `await chat()` call that exceeds the 5min wallclock window can still
|
||||
hit it. Fix: `RETURNING id` on the UPDATE + check `rows.length === 0` →
|
||||
throw tagged `LockLostError`. Phases catch + abort cleanly (write partial
|
||||
progress, return `status: 'fail'` with reason `'lock_lost'`). Behavioral
|
||||
contract change with phase-abort fallout; needs its own design pass.
|
||||
- [x] **TODO-OPS-2 (P2)**: Lock-loss detection — CLOSED by the W0 fix-wave
|
||||
(code-smell series). `refresh()` now runs a FENCED update (id + holder_pid +
|
||||
epoch-rendered `acquired_at`) with `RETURNING id`, returns `false` on 0
|
||||
rows, and runCycle's steal controller aborts the run at the next boundary
|
||||
with a structured `reason: 'lock_stolen'` partial report (LockStolenError;
|
||||
raced awaits cover the 5 long phases). The supervisor exits LOCK_LOST
|
||||
immediately on a fenced miss. Pinned by `test/db-lock-fencing.test.ts` +
|
||||
`test/cycle-lock-steal.serial.test.ts`.
|
||||
|
||||
## v0.41.20.0 status + doctor-categories wave follow-ups (v0.42+)
|
||||
|
||||
|
||||
+72
-4
@@ -7,17 +7,76 @@ only.
|
||||
|
||||
### Test command tiers
|
||||
|
||||
Seven test command tiers, each with a clear scope:
|
||||
Six test command tiers, each with a clear scope:
|
||||
|
||||
| Command | What it runs | Wallclock | When to use |
|
||||
|---|---|---|---|
|
||||
| `bun run test` | Parallel unit-test fast loop. Sharded fan-out via `scripts/run-unit-parallel.sh` (default 4 shards — CPU-detected, clamped to a max of 8; 4 matches CI's fan-out and avoids PGLite WASM-init contention), then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. Memory-safe by default: total concurrency (shards × intra-shard files) is capped to available memory at `GBRAIN_TEST_MEM_PER_FILE_MB` (default 1536 — a PGLite WASM instance) per concurrent file, and two phantom-failure classes are automatically re-run serially (the rescue pass): failures carrying the WASM out-of-memory signature, and shards killed externally (SIGTERM/SIGKILL well before the shard timeout — sibling workspaces' process cleanup, memory jetsam). Phantoms pass serially and the run goes green with an `oom_rescued` note; real failures fail again serially and stay red. Knobs: `GBRAIN_TEST_NO_MEM_ADAPT=1`, `GBRAIN_TEST_NO_OOM_FALLBACK=1`, `GBRAIN_TEST_MAX_CONCURRENCY` (intra-shard, default 4), `GBRAIN_TEST_SHARD_TIMEOUT` / `GBRAIN_TEST_SHARD_KILL_AFTER`, plus `--shards N` / `--max-concurrency N` / `--dry-run` script args. | a few minutes on a Mac dev box | Inner edit loop. Default. |
|
||||
| `bun run test` | Parallel unit-test fast loop. Sharded fan-out via `scripts/run-unit-parallel.sh` (default 4 shards — CPU-detected, clamped to a max of 8; 4 matches CI's fan-out and avoids PGLite WASM-init contention), then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. Builds/refreshes the PGLite schema snapshot BEFORE the shard fan-out and exports `GBRAIN_PGLITE_SNAPSHOT` so PGLite-booting files restore a baked schema instead of replaying every migration (~10x wallclock on a full run; see "PGLite schema snapshot" below). Opt out: `GBRAIN_NO_SNAPSHOT=1`. Memory-safe by default: total concurrency (shards × intra-shard files) is capped to available memory at `GBRAIN_TEST_MEM_PER_FILE_MB` (default 1536 — a PGLite WASM instance) per concurrent file, and two phantom-failure classes are automatically re-run serially (the rescue pass): failures carrying the WASM out-of-memory signature, and shards killed externally (SIGTERM/SIGKILL well before the shard timeout — sibling workspaces' process cleanup, memory jetsam). Phantoms pass serially and the run goes green with an `oom_rescued` note; real failures fail again serially and stay red. Knobs: `GBRAIN_TEST_NO_MEM_ADAPT=1`, `GBRAIN_TEST_NO_OOM_FALLBACK=1`, `GBRAIN_TEST_MAX_CONCURRENCY` (intra-shard, default 4), `GBRAIN_TEST_SHARD_TIMEOUT` / `GBRAIN_TEST_SHARD_KILL_AFTER`, plus `--shards N` / `--max-concurrency N` / `--dry-run` script args. | a few minutes on a Mac dev box | Inner edit loop. Default. |
|
||||
| `bun run verify` | CI's authoritative pre-test gate set, fanned out in parallel by `scripts/run-verify-parallel.sh`: the full `check:*` battery (privacy, jsonb, progress, source-id, test-isolation, wasm, …) plus `bun run typecheck`. The `CHECKS` array in that script is the single source of truth — CI literally calls `bun run verify` in a dedicated job. | ~16s (parallel; typecheck dominates) | Before pushing; before `/ship`. |
|
||||
| `bun run test:full` | `verify && bun run test && bun run test:slow && [smart e2e]`. The local equivalent of "everything CI runs." Smart e2e: runs e2e only when `DATABASE_URL` is set; else loud skip notice to stderr. | ~3-5min depending on slow + e2e | Pre-merge sanity, before opening a PR. |
|
||||
| `bun run test:slow` | Just the `*.slow.test.ts` set (intentional cold-path correctness checks). | seconds-to-minutes | When touching slow-path code. |
|
||||
| `bun run test:serial` | Just the `*.serial.test.ts` set (cross-file-contention quarantine; one bun process per file for true module-registry isolation). | ~1s per quarantined file | Debugging a specific quarantined file. |
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential. | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | The historical pre-check scripts (chained sequentially in package.json). Overlaps `verify` heavily but is NOT a superset — `verify`'s `CHECKS` array in `scripts/run-verify-parallel.sh` is the authoritative gate; `check:all` keeps a few local-only extras (trailing-newline, exports-count, no-legacy-getconnection). | ~10s | Local-only sweep for the extras. |
|
||||
|
||||
There is no `check:all` script anymore — it was a second, hand-synced guard
|
||||
registry that drifted from `verify` (three checks were reachable ONLY from it,
|
||||
i.e. never ran anywhere). The `CHECKS` array in `scripts/run-verify-parallel.sh`
|
||||
is the single execution list, and it now includes the former `check:all`-only
|
||||
extras (`check:newlines`, `check:exports-count`, `check:no-legacy-getconnection`).
|
||||
The guard REGISTRY is `scripts/guards-manifest.tsv` (see "Guard registry and
|
||||
self-test" below).
|
||||
|
||||
### PGLite schema snapshot (default-on)
|
||||
|
||||
`scripts/build-pglite-snapshot.ts` (`bun run build:pglite-snapshot`) bakes a
|
||||
post-`initSchema()` PGLite data dir into `test/fixtures/pglite-snapshot.tar`
|
||||
plus a version file; `PGLiteEngine.initSchema()` restores the tar instead of
|
||||
replaying the embedded schema + all migrations when the env var
|
||||
`GBRAIN_PGLITE_SNAPSHOT` points at it. Both `bun run test`
|
||||
(`scripts/run-unit-parallel.sh`, before the shard fan-out) and
|
||||
`scripts/ci-local.sh` call the builder unconditionally and export the env var.
|
||||
Measured effect: a full parallel suite run drops ~10x (PGLite-booting files go
|
||||
~1.63s → ~0.91s each). Properties:
|
||||
|
||||
- **Idempotent.** A hash short-circuit exits in ~40ms when the snapshot is
|
||||
fresh, and REBUILDS a stale one. The hash covers `PGLITE_SCHEMA_SQL`, every
|
||||
migration's `sql` + `sqlFor.pglite`, AND each migration `handler`'s function
|
||||
source (`Function.prototype.toString`) — 19+ migrations carry executable
|
||||
handler code with empty `sql` that a sql-only hash cannot see.
|
||||
- **Concurrency-safe.** Parallel shard runners / sibling workspaces serialize
|
||||
on an atomic `mkdir` lock (`test/fixtures/.pglite-snapshot.lock`) with
|
||||
staleness-verified takeover of a crashed builder; the tar is written first
|
||||
and the version file last, so a crash can never leave a fresh-looking torn
|
||||
fixture. `GBRAIN_SNAPSHOT_LOCK_TIMEOUT_MS` (default 120000) bounds the
|
||||
waiter; an exhausted waiter facing a still-live lock proceeds unlocked as a
|
||||
last resort (the loader gate below validates the version file, not the tar
|
||||
bytes).
|
||||
- **Never authoritative.** The loader (`tryLoadSnapshot` in
|
||||
`src/core/pglite-engine.ts`) verifies the schema hash AND the embedding
|
||||
shape the snapshot was baked with (`dims=` / `model=` lines in the version
|
||||
file) against what this process would create; any mismatch — including a
|
||||
version file without shape lines — warns once and falls through to normal
|
||||
cold init. A wrong fixture can never poison the suite.
|
||||
- **Opt out.** `GBRAIN_NO_SNAPSHOT=1` skips the build + env export for a run;
|
||||
the migration-replay canary tests clear the env themselves regardless.
|
||||
|
||||
Pinned by `test/snapshot-shape-guard.test.ts` (hash + shape refusal matrix,
|
||||
handler-source hash sensitivity).
|
||||
|
||||
### Guard registry and self-test
|
||||
|
||||
`scripts/guards-manifest.tsv` is THE single registry of `scripts/check-*`
|
||||
guards (currently 45), each classified `scanner` (greps/parses repo sources —
|
||||
must eventually carry fixtures), `buildfresh`, or `repostate` (build/freshness
|
||||
guards are exempt-with-reason, not fixture-tested).
|
||||
`scripts/guard-self-test.sh` (`bun run check:guard-self-test`, wired into
|
||||
`bun run verify`) proves every `selftest=yes` scanner CAN fail: it runs each
|
||||
one against known-bad (must exit non-zero) and known-good (must pass) fixture
|
||||
trees under `test/fixtures/guards/<guard>/{bad,good}/` via the
|
||||
`GBRAIN_GUARD_ROOT` env seam, and enforces manifest completeness — a new
|
||||
`scripts/check-*` script that isn't registered in the manifest fails the
|
||||
build. A guard whose pattern rots into a permanently-green no-op now fails CI
|
||||
instead of masquerading as coverage.
|
||||
|
||||
### Shell dispatch and Windows
|
||||
|
||||
@@ -90,7 +149,7 @@ Any change under `skills/` must regenerate it: `bun run scripts/generate-skills-
|
||||
|
||||
**This section is the canonical home of the test-isolation discipline** — CONTRIBUTING.md and other docs link here rather than restating the rules.
|
||||
|
||||
The cross-file flake class is enforced statically by `scripts/check-test-isolation.sh`, wired into `bun run verify` and `bun run check:all`. Rules (non-serial unit files only; `*.serial.test.ts` and `test/e2e/*` are skipped):
|
||||
The cross-file flake class is enforced statically by `scripts/check-test-isolation.sh`, wired into `bun run verify`. Rules (non-serial unit files only; `*.serial.test.ts` and `test/e2e/*` are skipped):
|
||||
|
||||
| Rule | What it bans | Fix |
|
||||
|---|---|---|
|
||||
@@ -264,6 +323,15 @@ Unit tests and what they cover:
|
||||
- `test/longmemeval-sanitize.test.ts` — sanitization parity pinning that `INJECTION_PATTERNS` from `src/core/think/sanitize.ts` is the single source of truth (adding a pattern there must cover both `<take>` framing and `<chat_session>` framing, no per-surface regex drift).
|
||||
- `test/openai-compat-multimodal.test.ts` — gateway's openai-compatible multimodal path: happy-path single + multi-input embedding, unauthenticated proxy mode, dimension-mismatch guard (throws `AIConfigError` with model id + observed + expected pre-storage), default-dim fallback when recipe declares `default_dims`, HTTP 401 / 400 / malformed-JSON / non-array error paths, regression that the existing Voyage `/multimodalembeddings` recipe still routes through its dedicated path. Hermetic via the `__setEmbedTransportForTests` seam.
|
||||
- `test/serve-stdio-lifecycle.test.ts` — `MCP_STDIO=1` env guard: stdin EOF does NOT trigger shutdown when the env is set, SIGTERM still does (guard scope is correct), unset env preserves the CLI lifecycle. Exercises the `ServeOptions.mcpStdio?: boolean` test seam directly so tests don't mutate `process.env`.
|
||||
- `test/db-lock-fencing.test.ts` — fenced lock identity: a `DbLockHandle` carries its acquisition fence, `refresh()` returns true while owned and false after a steal (0-row fenced UPDATE), a stolen-from handle's `release()` is a fenced no-op that leaves the successor's row intact, and `startCycleLockRefresher` aborts its controller with `LockStolenError` on a fenced miss while serializing ticks (a slow refresh never overlaps the next).
|
||||
- `test/cycle-lock-steal.serial.test.ts` — runCycle steal-abort arc end-to-end: a mid-run steal produces a structured partial report (`reason: 'lock_stolen'`), runs no further phases, and never touches the successor's lock row; a steal-free cycle completes and releases normally.
|
||||
- `test/cycle-any-abort-signal.test.ts` — `anyAbortSignal` combining: pre-aborted inputs, late aborts propagating their reason, duck-typed signal stubs (no `addEventListener`) observed via poll, and `dispose()` detaching the caller-signal listener + clearing the poll timer (the daemon leak class).
|
||||
- `test/queue-stall-parent-unblock.test.ts` — the shared `killJobs` tail: a stall-exhausted child lands `child_done(dead)` in its parent's inbox and unblocks the parent, a requeued child doesn't touch the parent, all three reapers route through the tail with their own outcome, and the idempotent stranded-parent sweep self-heals parents whose children were already dead (without unblocking parents that still have a live child).
|
||||
- `test/queue-started-at-retry.test.ts` — every automatic re-run path clears `started_at` (failJob delayed branch, stall requeue, lease release, promoteDelayed, parent re-claim) so a retried job's wall-clock budget measures execution, not backoff wait; end-to-end survival of the wall-clock sweep on a fresh attempt.
|
||||
- `test/embed-modality-preserved.test.ts` — `carryChunkMetadata` carries modality + all code-metadata fields through re-embed merges (an image chunk stays image), plus the write-side contract that omitting modality resets it to text (why the shared list is load-bearing).
|
||||
- `test/import-abort-error.test.ts` — `runImport` preflight/argv failures throw typed `ImportAbortError` instead of exiting the process; the calling process survives the abort.
|
||||
- `test/lint-fix-single-pass.test.ts` — `gbrain lint --fix` walks the tree once and `total_fixed` reports the fixes THIS run applied.
|
||||
- `test/snapshot-shape-guard.test.ts` — PGLite snapshot loader refusal matrix: shape-less version files, dims/model mismatches, and stale schema hashes are all refused; matching hash + shape loads; a migration-handler edit changes the hash.
|
||||
|
||||
### E2E test inventory
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,61 @@
|
||||
# Fix-wave series baselines (W0 → W9)
|
||||
|
||||
Recorded per wave so the series' "10x better for 2x effort" claim is measured,
|
||||
not vibed (fix-wave plan D4.13). Update this file in each wave's PR; keep the
|
||||
prior rows — the deltas ARE the receipt.
|
||||
|
||||
## How to refresh
|
||||
|
||||
```bash
|
||||
wc -l src/commands/doctor.ts src/core/pglite-engine.ts src/core/postgres-engine.ts \
|
||||
src/core/operations.ts src/core/migrate.ts src/commands/sync.ts \
|
||||
src/core/ai/gateway.ts src/cli.ts src/core/engine.ts \
|
||||
src/core/search/hybrid.ts src/core/search/mode.ts src/core/cycle.ts
|
||||
ls scripts/check-* | wc -l # guard count
|
||||
bash scripts/guard-self-test.sh # self-tested count + harness runtime
|
||||
bun run test > /tmp/suite.txt 2>&1; echo $? # wall-clock from the run banner
|
||||
```
|
||||
|
||||
Retrieval-quality canary (MANDATORY before W1, and after W1/W3/W9): run
|
||||
`gbrain eval gate` against a NON-PRODUCTION brain (the production PGLite brain
|
||||
is single-writer and usually held by a live `gbrain serve`; eval runs never
|
||||
touch `~/.gbrain` per the eval discipline — results land in
|
||||
`<repo>/.gbrain-evals/eval-results.jsonl`). Record the gate verdict + headline
|
||||
metrics here per run.
|
||||
|
||||
## W0 (2026-08-14, branch garrytan/code-smell-fix-wave @ post-hotfix)
|
||||
|
||||
God-file line counts (the audit's structural targets, BEFORE the registry waves):
|
||||
|
||||
| File | Lines |
|
||||
|---|---|
|
||||
| src/commands/doctor.ts | 10,057 |
|
||||
| src/core/operations.ts | 7,459 |
|
||||
| src/core/pglite-engine.ts | 6,874 |
|
||||
| src/core/postgres-engine.ts | 6,847 |
|
||||
| src/core/migrate.ts | 6,201 |
|
||||
| src/commands/sync.ts | 5,991 |
|
||||
| src/core/ai/gateway.ts | 4,049 |
|
||||
| src/cli.ts | 3,301 |
|
||||
| src/core/cycle.ts | 2,933 |
|
||||
| src/core/search/hybrid.ts | 2,453 |
|
||||
| src/core/engine.ts | 2,320 |
|
||||
| src/core/search/mode.ts | 1,232 |
|
||||
|
||||
Guards: 47 scripts/check-* files; 3 self-tested (harness <1s, budget 30s);
|
||||
single registry established (guards-manifest.tsv; `check:all` deleted; 3
|
||||
previously-unreachable guards wired into verify).
|
||||
|
||||
Test infra: PGLite snapshot default-on for `bun run test`. Per-PGLite-file:
|
||||
1.63s cold → 0.91s snapshotted (measured on test/db-lock-fencing.test.ts).
|
||||
Full-suite wall-clock (post-snapshot): recorded in the W0 ship notes — see
|
||||
the run banner of the W0 PR's `bun run test` evidence.
|
||||
|
||||
Retrieval canary: NOT RUN at W0 (production brain locked by live serve; W0
|
||||
touches no search paths). REQUIRED before W1 lands.
|
||||
|
||||
Verified-bug status at W0 ship: cycle-lock refresh + fencing (TODO-OPS-2
|
||||
closed), stall-death parent unblock, started_at ×4, modality carry, import
|
||||
typed aborts, lint single-pass, prompt EOF safety, guard self-test harness,
|
||||
snapshot default-on. W0a superseded by master's WP1/D7 (port-ledger in the
|
||||
plan file).
|
||||
+5
-1
@@ -225,7 +225,11 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
imports use static top-level imports. Besides the snapshot loader's lazy
|
||||
`require()` cluster in `pglite-engine.ts:tryLoadSnapshot` (fs/crypto/
|
||||
migrate/pglite-schema + one gateway shape lookup — lazy so production
|
||||
builds without the test-fixture path don't eager-load; the guard now
|
||||
matches `require()` calls too), the only dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.45.15.0",
|
||||
"version": "0.45.16.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
|
||||
+4
-3
@@ -50,7 +50,6 @@
|
||||
"check:admin-scope-drift": "bash scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "bash scripts/check-cli-executable.sh",
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:all": "bash scripts/check-privacy.sh && bash scripts/check-proposal-pii.sh && bash scripts/check-test-real-names.sh && bash scripts/check-jsonb-pattern.sh && bash scripts/check-source-id-projection.sh && bash scripts/check-source-config-leak.sh && bash scripts/check-progress-to-stdout.sh && bash scripts/check-no-tracked-symlinks.sh && bash scripts/check-no-legacy-getconnection.sh && bash scripts/check-test-isolation.sh && bash scripts/check-trailing-newline.sh && bash scripts/check-wasm-embedded.sh && bash scripts/check-pglite-embedded.sh && bash scripts/check-exports-count.sh && bash scripts/check-admin-build.sh && bash scripts/check-admin-scope-drift.sh && bash scripts/check-cli-executable.sh && bash scripts/check-skill-brain-first.sh && bash scripts/check-operations-filter-bypass.sh && bash scripts/check-gateway-routed-no-direct-anthropic.sh && bash scripts/check-worker-pool-atomicity.sh && bash scripts/check-key-files-current-state.sh && bash scripts/check-no-double-retry.sh && bash scripts/check-batch-audit-site.sh && bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:gateway-routed": "bash scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "bash scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "bash scripts/check-key-files-current-state.sh",
|
||||
@@ -97,7 +96,9 @@
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin",
|
||||
"check:skill-refs": "bun scripts/check-skill-refs.mjs",
|
||||
"gate:skills": "bash scripts/skills-commit-gate.sh"
|
||||
"gate:skills": "bash scripts/skills-commit-gate.sh",
|
||||
"check:guard-self-test": "bash scripts/guard-self-test.sh",
|
||||
"check:no-legacy-getconnection": "bash scripts/check-no-legacy-getconnection.sh"
|
||||
},
|
||||
"openclaw": {
|
||||
"compat": {
|
||||
@@ -155,7 +156,7 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.45.15.0",
|
||||
"version": "0.45.16.0",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.5",
|
||||
|
||||
@@ -18,10 +18,14 @@
|
||||
//
|
||||
// Re-run whenever you touch src/core/migrate.ts or src/schema.sql.
|
||||
|
||||
import { writeFileSync, mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { writeFileSync, mkdirSync, existsSync, readFileSync, rmdirSync, rmSync, mkdtempSync, statSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import * as crypto from "node:crypto";
|
||||
|
||||
import { configureGateway, getEmbeddingDimensions, getEmbeddingModel } from "../src/core/ai/gateway.ts";
|
||||
import { LEGACY_EMBEDDING_CONFIG } from "../test/helpers/legacy-embedding-config.ts";
|
||||
|
||||
import { PGLiteEngine, computeSnapshotSchemaHash } from "../src/core/pglite-engine.ts";
|
||||
import { MIGRATIONS } from "../src/core/migrate.ts";
|
||||
import { PGLITE_SCHEMA_SQL } from "../src/core/pglite-schema.ts";
|
||||
@@ -33,9 +37,84 @@ function computeSchemaHash(): string {
|
||||
async function main() {
|
||||
const fixturePath = "test/fixtures/pglite-snapshot.tar";
|
||||
const versionPath = "test/fixtures/pglite-snapshot.version";
|
||||
const lockPath = "test/fixtures/.pglite-snapshot.lock";
|
||||
mkdirSync(dirname(fixturePath), { recursive: true });
|
||||
|
||||
// W0 fix-wave: build under the EXACT embedding shape the test suite pins.
|
||||
// bunfig.toml preloads test/helpers/legacy-embedding-preload.ts, which
|
||||
// configures the gateway to the shared LEGACY_EMBEDDING_CONFIG (OpenAI
|
||||
// 1536-d) for every `bun test` file — so the snapshot's baked vector(dims)
|
||||
// columns MUST match that shape, not the builder machine's ambient config
|
||||
// (nor the shipped 1280-d default an unconfigured gateway falls back to).
|
||||
// Set in main(), not module scope: ESM hoists imports, so module-scope
|
||||
// placement implied an ordering it never had — config reads are lazy.
|
||||
configureGateway({ ...LEGACY_EMBEDDING_CONFIG, env: { ...process.env } });
|
||||
|
||||
const schemaHash = computeSchemaHash();
|
||||
|
||||
// W0 fix-wave (Tier-1 #16): idempotent short-circuit. Runners now call this
|
||||
// script UNCONDITIONALLY (build-if-missing left stale-but-present snapshots
|
||||
// permanently on the warn+slow path); a fresh snapshot exits in ~ms.
|
||||
const isFresh = () => {
|
||||
if (!existsSync(fixturePath) || !existsSync(versionPath)) return false;
|
||||
const lines = readFileSync(versionPath, "utf-8").trim().split("\n");
|
||||
return lines[0] === schemaHash
|
||||
&& lines[1] === `dims=${getEmbeddingDimensions()}`
|
||||
&& lines[2] === `model=${getEmbeddingModel()}`;
|
||||
};
|
||||
if (isFresh()) {
|
||||
console.log(`[build-pglite-snapshot] up to date (hash ${schemaHash.slice(0, 16)}...) — nothing to do`);
|
||||
return;
|
||||
}
|
||||
|
||||
// GBRAIN_HOME isolation is only needed once we actually BUILD (the engine
|
||||
// boot reads config). Red-team catch: creating it before the isFresh()
|
||||
// short-circuit leaked one temp dir per invocation on the COMMON path
|
||||
// (this script runs on every `bun run test`).
|
||||
const hermeticHome = mkdtempSync(join(tmpdir(), "gbrain-snapshot-hermetic-"));
|
||||
process.env.GBRAIN_HOME = hermeticHome;
|
||||
|
||||
// W0 fix-wave (D5.8): concurrency lock. Parallel shard runners / concurrent
|
||||
// Conductor workspaces invoking this simultaneously must not tear the tar.
|
||||
// mkdir is atomic; the loser polls until the winner finishes, then
|
||||
// re-checks freshness and exits.
|
||||
let ownLock = false;
|
||||
try {
|
||||
mkdirSync(lockPath);
|
||||
ownLock = true;
|
||||
} catch {
|
||||
console.log(`[build-pglite-snapshot] another builder holds ${lockPath}; waiting...`);
|
||||
const timeoutMs = Number(process.env.GBRAIN_SNAPSHOT_LOCK_TIMEOUT_MS) || 120_000;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (existsSync(lockPath) && Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 250));
|
||||
}
|
||||
if (isFresh()) {
|
||||
console.log(`[build-pglite-snapshot] concurrent builder finished; snapshot fresh`);
|
||||
return;
|
||||
}
|
||||
// Stale lock (crashed builder) or still-stale snapshot: TAKE OVER.
|
||||
// W0 ship-review catch: mkdirSync on a still-existing dir always throws
|
||||
// EEXIST — the original retry could never acquire, so a single crashed
|
||||
// builder left every future rebuild waiting the full deadline and then
|
||||
// proceeding UNLOCKED forever (the stale dir was never removed).
|
||||
// Red-team refinement: verify STALENESS (lock dir mtime older than the
|
||||
// full wait window) before the rmdir — two exhausted waiters would
|
||||
// otherwise each rmdir+mkdir and the second would steal the first's
|
||||
// just-created LIVE lock, re-opening the torn-tar window.
|
||||
try {
|
||||
if (existsSync(lockPath)) {
|
||||
const ageMs = Date.now() - statSync(lockPath).mtimeMs;
|
||||
if (ageMs > timeoutMs) {
|
||||
console.log(`[build-pglite-snapshot] stale lock (age ${Math.round(ageMs / 1000)}s > ${Math.round(timeoutMs / 1000)}s) — taking over`);
|
||||
rmdirSync(lockPath);
|
||||
}
|
||||
}
|
||||
mkdirSync(lockPath);
|
||||
ownLock = true;
|
||||
} catch { /* lock is LIVE (fresh mtime) or takeover raced; proceed unlocked as last resort */ }
|
||||
}
|
||||
try {
|
||||
console.log(`[build-pglite-snapshot] schema hash: ${schemaHash.slice(0, 16)}...`);
|
||||
console.log(`[build-pglite-snapshot] booting PGLite (in-memory)...`);
|
||||
const engine = new PGLiteEngine();
|
||||
@@ -53,12 +132,21 @@ async function main() {
|
||||
const dump = await engine.db.dumpDataDir("none");
|
||||
const buffer = Buffer.from(await dump.arrayBuffer());
|
||||
|
||||
// Write tar first, version LAST — the version file is the commit point, so
|
||||
// a crash between the writes leaves a stale-hash (ignored) snapshot, never
|
||||
// a fresh-looking torn one. Lines 2-3 record the embedding shape the
|
||||
// snapshot was baked with; the loader refuses a shape-mismatched snapshot
|
||||
// (the W0 1280-vs-1536 incident class).
|
||||
writeFileSync(fixturePath, buffer);
|
||||
writeFileSync(versionPath, schemaHash + "\n");
|
||||
writeFileSync(versionPath, `${schemaHash}\ndims=${getEmbeddingDimensions()}\nmodel=${getEmbeddingModel()}\n`);
|
||||
await engine.disconnect();
|
||||
|
||||
console.log(`[build-pglite-snapshot] wrote ${fixturePath} (${buffer.length} bytes)`);
|
||||
console.log(`[build-pglite-snapshot] wrote ${versionPath}`);
|
||||
} finally {
|
||||
if (ownLock) { try { rmdirSync(lockPath); } catch { /* best effort */ } }
|
||||
try { rmSync(hermeticHome, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# scopes to admin/src/ so we can't import the source list directly; instead
|
||||
# this script extracts both lists and diffs them.
|
||||
#
|
||||
# Wired into `bun run verify` and `bun run check:all`.
|
||||
# Wired into `bun run verify` (single guard registry: scripts/guards-manifest.tsv).
|
||||
#
|
||||
# Exits 0 on match, 1 on drift, 2 on internal error (file missing, parse fail).
|
||||
#
|
||||
|
||||
@@ -49,7 +49,16 @@ for (const file of files) {
|
||||
}
|
||||
|
||||
function visit(node: ts.Node): void {
|
||||
if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) {
|
||||
// W0 ship-review catch: match BOTH lazy-loading forms. The guard
|
||||
// previously matched only `import(...)` call expressions, so a
|
||||
// `require(...)` on an engine-live path passed silently and its
|
||||
// engine-dynamic-import-ok marker was decorative.
|
||||
const isDynamicImport = ts.isCallExpression(node)
|
||||
&& node.expression.kind === ts.SyntaxKind.ImportKeyword;
|
||||
const isRequireCall = ts.isCallExpression(node)
|
||||
&& ts.isIdentifier(node.expression)
|
||||
&& node.expression.text === 'require';
|
||||
if (isDynamicImport || isRequireCall) {
|
||||
const { line } = sourceFile.getLineAndCharacterOfPosition(node.expression.getStart(sourceFile));
|
||||
const sourceLine = lines[line] ?? '';
|
||||
if (!markerLines.has(line)) {
|
||||
|
||||
@@ -17,26 +17,43 @@ set -euo pipefail
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Match the interpolated form: ${JSON.stringify(...)}::jsonb
|
||||
# Using grep -P for Perl-compatible regex (lookahead-free pattern is enough here).
|
||||
PATTERN='\$\{JSON\.stringify\([^)]*\)\}::jsonb'
|
||||
# W0 fix-wave (Tier-1 #11): self-test seam — the guard harness points this at
|
||||
# a known-bad fixture tree and asserts exit 1.
|
||||
SCAN_ROOT="${GBRAIN_GUARD_ROOT:-src/}"
|
||||
|
||||
if grep -rEn "$PATTERN" src/ 2>/dev/null; then
|
||||
# Match the interpolated form: ${JSON.stringify(...)}::jsonb
|
||||
#
|
||||
# W0 fix-wave (Tier-1 #11): the previous `\([^)]*\)` argument matcher could
|
||||
# not cross a nested `)` — `${JSON.stringify(obj.get())}::jsonb` was
|
||||
# invisible (the same regex-hole class that made check-no-double-retry a
|
||||
# permanently-green no-op). `[^}]*` spans nested parens but CANNOT cross the
|
||||
# interpolation's closing `}`, so a safe `${JSON.stringify(x)}::text::jsonb`
|
||||
# followed by a separate `${expr()}::jsonb` on the same line is not spanned
|
||||
# into a false positive (ship-review catch — the greedy `.*` variant was).
|
||||
PATTERN='\$\{JSON\.stringify\([^}]*\)\}::jsonb'
|
||||
|
||||
if grep -rEn "$PATTERN" "$SCAN_ROOT" 2>/dev/null; then
|
||||
echo
|
||||
echo "ERROR: Found JSON.stringify(...)::jsonb pattern in src/."
|
||||
echo "ERROR: Found JSON.stringify(...)::jsonb pattern in $SCAN_ROOT."
|
||||
echo " postgres.js v3 stringifies again, producing JSONB string literals."
|
||||
echo " Use sql.json(x) instead. See feedback_postgres_jsonb_double_encode.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: no JSON.stringify(x)::jsonb interpolation pattern in src/"
|
||||
echo "OK: no JSON.stringify(x)::jsonb interpolation pattern in $SCAN_ROOT"
|
||||
|
||||
# v0.13.1 #219: guard against max_stalled DEFAULT 1 regressing in any schema
|
||||
# source file. DEFAULT 1 dead-lettered any SIGKILL'd job on first stall, making
|
||||
# the "10/10 rescued" claim false for out-of-the-box users. Default is 5 now.
|
||||
MAX_STALLED_PATTERN='max_stalled\s+INTEGER\s+NOT\s+NULL\s+DEFAULT\s+1\b'
|
||||
|
||||
if grep -rEn "$MAX_STALLED_PATTERN" src/schema.sql src/core/migrate.ts src/core/pglite-schema.ts src/core/schema-embedded.ts 2>/dev/null; then
|
||||
# Schema files are fixed paths; under a fixture root (self-test) they don't
|
||||
# exist — skip rather than fail on the missing-file grep.
|
||||
SCHEMA_FILES=()
|
||||
for f in src/schema.sql src/core/migrate.ts src/core/pglite-schema.ts src/core/schema-embedded.ts; do
|
||||
[ -f "$f" ] && SCHEMA_FILES+=("$f")
|
||||
done
|
||||
if [ "${#SCHEMA_FILES[@]}" -gt 0 ] && grep -rEn "$MAX_STALLED_PATTERN" "${SCHEMA_FILES[@]}" 2>/dev/null; then
|
||||
echo
|
||||
echo "ERROR: max_stalled DEFAULT 1 reintroduced in schema."
|
||||
echo " Must be DEFAULT 5 to preserve SIGKILL-rescue guarantee. See #219."
|
||||
@@ -51,10 +68,11 @@ echo "OK: max_stalled defaults are 5 in all schema sources"
|
||||
# [JSON.stringify(x)]) — which is the exact shape that double-encoded the
|
||||
# op_checkpoints pin and aborted every sync in #2339. The AST-lite scanner below
|
||||
# catches it. `set -e` propagates its non-zero exit.
|
||||
# Under a fixture root, scan that root; the AST-lite scanner takes roots as argv.
|
||||
if command -v node >/dev/null 2>&1; then
|
||||
node scripts/check-jsonb-params.mjs
|
||||
node scripts/check-jsonb-params.mjs ${GBRAIN_GUARD_ROOT:+"$GBRAIN_GUARD_ROOT"}
|
||||
elif command -v bun >/dev/null 2>&1; then
|
||||
bun scripts/check-jsonb-params.mjs
|
||||
bun scripts/check-jsonb-params.mjs ${GBRAIN_GUARD_ROOT:+"$GBRAIN_GUARD_ROOT"}
|
||||
else
|
||||
echo "WARN: neither node nor bun on PATH; skipping check-jsonb-params.mjs" >&2
|
||||
fi
|
||||
|
||||
@@ -19,16 +19,24 @@ set -euo pipefail
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# W0 fix-wave (Tier-1 #11): self-test seam. The guard harness points this at
|
||||
# a known-bad fixture tree and asserts exit 1 — the guard can no longer rot
|
||||
# into a permanently-green no-op unnoticed.
|
||||
SCAN_ROOT="${GBRAIN_GUARD_ROOT:-src/}"
|
||||
|
||||
# Match: withRetry(...) wrapping any of the 3 engine batch methods.
|
||||
# The greedy `.*` between `withRetry(` and `engine.` covers both the
|
||||
# arrow-fn form and any direct invocation. (gbrain-allow-direct-insert: doc comment)
|
||||
# Multi-line wraps are caught by `grep -E` per file (line-wise) for the
|
||||
# common single-line case; multi-line wraps still get caught by a separate
|
||||
# multi-line pass below.
|
||||
PATTERN='withRetry\([^)]*engine\.(addLinksBatch|addTimelineEntriesBatch|upsertChunks)'
|
||||
#
|
||||
# W0 fix-wave (Tier-1 #11): the previous pattern used `[^)]*` between
|
||||
# `withRetry(` and `engine.`, which can never cross the `)` in `() =>` — so
|
||||
# the CANONICAL banned shape (an arrow function wrapping the engine batch
|
||||
# call) was invisible and the guard had been permanently green since it
|
||||
# shipped. `.*` (line-bounded by grep) covers the arrow form, async arrows,
|
||||
# and any argument shape. (Spelled without the literal call token here —
|
||||
# check-system-of-record scans scripts/ comments too: the prose-bleed class.)
|
||||
PATTERN='withRetry\(.*engine\.(addLinksBatch|addTimelineEntriesBatch|upsertChunks)'
|
||||
|
||||
# Single-line scan (covers ~95% of real cases).
|
||||
if grep -rEn "$PATTERN" src/ --include='*.ts' 2>/dev/null; then
|
||||
if grep -rEn "$PATTERN" "$SCAN_ROOT" --include='*.ts' 2>/dev/null; then
|
||||
echo
|
||||
echo "ERROR: Found withRetry(...engine.{addLinksBatch|addTimelineEntriesBatch|upsertChunks})"
|
||||
echo " pattern in src/."
|
||||
@@ -47,17 +55,26 @@ if grep -rEn "$PATTERN" src/ --include='*.ts' 2>/dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Multi-line scan: a withRetry( on one line and the engine call on the next
|
||||
# few. Bounded to 3-line window so we don't flag distant unrelated calls.
|
||||
# Uses pcregrep if available, else falls back to a simple awk window.
|
||||
if command -v pcregrep >/dev/null 2>&1; then
|
||||
if pcregrep -r -M -n --include='\.ts$' \
|
||||
'withRetry\([^)]*\n\s*\(?[^)]*=>\s*engine\.(addLinksBatch|addTimelineEntriesBatch|upsertChunks)' \
|
||||
src/ 2>/dev/null; then
|
||||
echo
|
||||
echo "ERROR: Multi-line withRetry(...engine.batch...) wrap found in src/. See above."
|
||||
exit 1
|
||||
fi
|
||||
# Multi-line scan: a withRetry( on one line and the engine call within the
|
||||
# next 3 lines. W0 fix-wave (Tier-1 #11): the previous pass was gated on
|
||||
# pcregrep, which is not installed on dev machines OR CI — it never ran.
|
||||
# perl is always available; same 3-line window, always on.
|
||||
#
|
||||
# Ship-review catch: perl must ALWAYS exit 0 and let OUTPUT PRESENCE decide.
|
||||
# An exit-1-from-clean-batches design breaks under `set -o pipefail` the
|
||||
# moment src/ outgrows one xargs batch (xargs exits 123, overriding grep's
|
||||
# verdict) — a silently missed violation, the same permanently-green class
|
||||
# this guard was just cured of.
|
||||
MULTILINE_MATCHES=$(find "$SCAN_ROOT" -name '*.ts' -type f -print0 2>/dev/null | xargs -0 perl -0777 -ne '
|
||||
if (/withRetry\([^\n]*\n(?:[^\n]*\n){0,2}?[^\n]*engine\.(?:addLinksBatch|addTimelineEntriesBatch|upsertChunks)/) {
|
||||
print "$ARGV: multi-line withRetry wrap around an engine batch call\n";
|
||||
}
|
||||
' 2>/dev/null || true)
|
||||
if [ -n "$MULTILINE_MATCHES" ]; then
|
||||
echo "$MULTILINE_MATCHES"
|
||||
echo
|
||||
echo "ERROR: Multi-line withRetry(...engine.batch...) wrap found in $SCAN_ROOT. See above."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: no withRetry(...engine.batch...) double-retry patterns in src/"
|
||||
echo "OK: no withRetry(...engine.batch...) double-retry patterns in $SCAN_ROOT"
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
# - new logging APIs that may show up later (the regex matches the URL,
|
||||
# not the consumer; any leak will trip)
|
||||
#
|
||||
# Wired into bun run check:all and bun run verify.
|
||||
# Wired into bun run verify (single guard registry: scripts/guards-manifest.tsv).
|
||||
#
|
||||
# Exit codes: 0 = clean, 1 = found at least one suspect line.
|
||||
set -euo pipefail
|
||||
|
||||
+6
-6
@@ -236,12 +236,12 @@ bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo \"[runner] Tier 3: building PGLite snapshot fixture (cached across reruns)\"
|
||||
if [ ! -f test/fixtures/pglite-snapshot.tar ] || [ ! -f test/fixtures/pglite-snapshot.version ]; then
|
||||
bun run build:pglite-snapshot
|
||||
else
|
||||
echo \"[runner] snapshot fixture exists; engine will validate hash at load time\"
|
||||
fi
|
||||
echo \"[runner] Tier 3: PGLite snapshot fixture (idempotent; rebuilds on hash drift)\"
|
||||
# W0 fix-wave (Tier-1 #16): unconditional call — the build script self-
|
||||
# short-circuits on a fresh hash and rebuilds STALE snapshots (the old
|
||||
# if-missing guard left a stale-but-present snapshot permanently on the
|
||||
# warn+slow path). Concurrency-safe via the script's mkdir lock (D5.8).
|
||||
bun run build:pglite-snapshot
|
||||
export GBRAIN_PGLITE_SNAPSHOT=test/fixtures/pglite-snapshot.tar
|
||||
echo \"[runner] resolving E2E file selection (--diff aware)\"
|
||||
${DIFF_E2E_PREP}
|
||||
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# W0 fix-wave (Tier-1 #11 / D5.14): guard self-test harness.
|
||||
#
|
||||
# The audit found scripts/check-no-double-retry.sh had been PERMANENTLY GREEN
|
||||
# since it shipped: its regex could not match the canonical banned shape, and
|
||||
# its multi-line fallback was gated on pcregrep, which is installed nowhere.
|
||||
# A guard that cannot fail is worse than no guard — it reads as coverage.
|
||||
#
|
||||
# This harness makes that class structurally impossible for scanner guards:
|
||||
# every guard marked `selftest yes` in scripts/guards-manifest.tsv is run
|
||||
# against test/fixtures/guards/<guard>/bad (MUST exit non-zero) and
|
||||
# .../good (MUST exit 0), via the GBRAIN_GUARD_ROOT override each guard
|
||||
# honors. Adding a self-test to a `todo` scanner = flip the manifest flag +
|
||||
# drop two fixture files.
|
||||
#
|
||||
# Also prints total harness wall-clock (guard-runtime budget line, D4.5):
|
||||
# fails if the self-test pass exceeds the budget, so guard sprawl shows up
|
||||
# here before it shows up as slow `bun run verify`.
|
||||
#
|
||||
# Usage: scripts/guard-self-test.sh
|
||||
# Exit: 0 = every self-tested guard fails on bad + passes on good.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
MANIFEST="scripts/guards-manifest.tsv"
|
||||
FIXTURES="test/fixtures/guards"
|
||||
BUDGET_SECONDS=30
|
||||
START=$(date +%s)
|
||||
failures=0
|
||||
tested=0
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "ERROR: $MANIFEST missing — the guard registry is load-bearing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_guard() {
|
||||
local guard="$1" fixture_root="$2"
|
||||
case "$guard" in
|
||||
*.mjs) GBRAIN_GUARD_ROOT="$fixture_root" node "scripts/$guard" "$fixture_root" >/dev/null 2>&1 ;;
|
||||
*) GBRAIN_GUARD_ROOT="$fixture_root" bash "scripts/$guard" >/dev/null 2>&1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
while IFS=$'\t' read -r guard klass selftest _notes; do
|
||||
case "$guard" in ''|'#'*) continue ;; esac
|
||||
[ "$selftest" = "yes" ] || continue
|
||||
tested=$((tested + 1))
|
||||
|
||||
bad="$FIXTURES/$guard/bad"
|
||||
good="$FIXTURES/$guard/good"
|
||||
if [ ! -d "$bad" ] || [ ! -d "$good" ]; then
|
||||
echo "FAIL $guard: manifest says selftest=yes but fixtures missing under $FIXTURES/$guard/{bad,good}"
|
||||
failures=$((failures + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if run_guard "$guard" "$bad"; then
|
||||
echo "FAIL $guard: did NOT flag the known-bad fixture — the guard is a no-op (the check-no-double-retry class)"
|
||||
failures=$((failures + 1))
|
||||
elif ! run_guard "$guard" "$good"; then
|
||||
echo "FAIL $guard: flagged the known-good fixture — false positive"
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
echo "ok $guard (bad→fail, good→pass)"
|
||||
fi
|
||||
done < "$MANIFEST"
|
||||
|
||||
# Manifest completeness: every scripts/check-* guard must have a manifest row
|
||||
# (new guards can't silently skip classification).
|
||||
for f in scripts/check-*.sh scripts/check-*.mjs; do
|
||||
base="$(basename "$f")"
|
||||
# The .ts companion of check-engine-dynamic-import is an implementation file.
|
||||
if ! grep -q "^${base} " "$MANIFEST"; then
|
||||
echo "FAIL $base: no row in $MANIFEST — classify it (scanner|buildfresh|repostate)"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
echo "guard self-test: $tested guard(s) self-tested, ${ELAPSED}s (budget ${BUDGET_SECONDS}s)"
|
||||
if [ "$ELAPSED" -gt "$BUDGET_SECONDS" ]; then
|
||||
echo "FAIL guard self-test exceeded the ${BUDGET_SECONDS}s runtime budget — trim fixtures or parallelize before adding more"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo "ERROR: $failures guard self-test failure(s)."
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: all self-tested guards catch their bad fixtures and pass their good ones"
|
||||
@@ -0,0 +1,62 @@
|
||||
# CI guard registry (W0 fix-wave, Tier-1 #11 / D5.14).
|
||||
# THE single registry of scripts/check-* guards. package.json's `check:all`
|
||||
# (a second, stale, hand-synced copy) was deleted; scripts/run-verify-parallel.sh
|
||||
# executes guards, and scripts/guard-self-test.sh consumes THIS file to
|
||||
# self-test every scanner guard against known-bad/known-good fixtures under
|
||||
# test/fixtures/guards/<guard>/{bad,good}/ (env: GBRAIN_GUARD_ROOT).
|
||||
#
|
||||
# class: scanner = greps/parses repo sources; MUST eventually carry fixtures
|
||||
# (selftest yes|todo). A scanner guard with selftest=todo is
|
||||
# tracked debt — the class that produced two permanently-
|
||||
# green guards (check-no-double-retry, pcregrep-gated pass).
|
||||
# buildfresh = runs builds/regenerators and diffs outputs; self-tests
|
||||
# don't apply (the build IS the test). exempt.
|
||||
# repostate = checks repo/file state (modes, symlinks, VERSION stamps);
|
||||
# exempt with reason.
|
||||
#
|
||||
# guard class selftest notes
|
||||
check-no-double-retry.sh scanner yes regex hole fixed in W0 (could not match `() =>`); perl multi-line pass replaces never-installed pcregrep
|
||||
check-jsonb-pattern.sh scanner yes nested-paren hole fixed in W0; safe ::text::jsonb spelling stays unflagged
|
||||
check-jsonb-params.mjs scanner yes positional $N::jsonb AST-lite scanner; argv/env root override
|
||||
check-batch-audit-site.sh scanner todo
|
||||
check-bun-test-timeout.sh scanner todo
|
||||
check-fixture-privacy.sh scanner todo
|
||||
check-no-legacy-getconnection.sh scanner todo was reachable from neither verify nor CI pre-W0 (check:all only)
|
||||
check-no-pii-in-agent-voice.sh scanner todo
|
||||
check-operations-filter-bypass.sh scanner todo
|
||||
check-pagetype-exhaustive.sh scanner todo
|
||||
check-pg-url-redaction.sh scanner todo
|
||||
check-privacy.sh scanner todo
|
||||
check-progress-to-stdout.sh scanner todo
|
||||
check-proposal-pii.sh scanner todo
|
||||
check-search-path.sh scanner todo
|
||||
check-skill-brain-first.sh scanner todo
|
||||
check-skill-refs.mjs scanner todo
|
||||
check-source-config-leak.sh scanner todo
|
||||
check-source-id-projection.sh scanner todo
|
||||
check-source-scope-onboard.sh scanner todo
|
||||
check-synthetic-corpus-privacy.sh scanner todo
|
||||
check-system-of-record.sh scanner todo
|
||||
check-test-real-names.sh scanner todo
|
||||
check-worker-lock-renewal-shape.sh scanner todo
|
||||
check-worker-pool-atomicity.sh scanner todo
|
||||
check-gateway-routed-no-direct-anthropic.sh scanner todo
|
||||
check-engine-dynamic-import.sh scanner todo .ts companion is its implementation, not a separate guard
|
||||
check-key-files-current-state.sh scanner todo
|
||||
check-exports-count.sh scanner todo was reachable from neither verify nor CI pre-W0 (check:all only)
|
||||
check-trailing-newline.sh scanner todo was reachable from neither verify nor CI pre-W0 (check:all only)
|
||||
check-test-isolation.sh scanner todo allowlist data file: check-test-isolation.allowlist
|
||||
check-admin-build.sh buildfresh exempt runs the admin build; the build is the test
|
||||
check-admin-embedded.sh buildfresh exempt embed freshness diff
|
||||
check-admin-scope-drift.sh buildfresh exempt regenerates + diffs
|
||||
check-bootstrap-templates.sh buildfresh exempt regenerates template tree + diffs
|
||||
check-eval-glossary-fresh.sh buildfresh exempt regenerates + diffs
|
||||
check-fuzz-purity.sh buildfresh exempt executes fuzz corpus
|
||||
check-image-decoders-embedded.sh buildfresh exempt binary embed check
|
||||
check-pglite-embedded.sh buildfresh exempt binary embed check
|
||||
check-skills-manifest-fresh.sh buildfresh exempt regenerates + diffs
|
||||
check-tool-catalog-fresh.sh buildfresh exempt regenerates + diffs
|
||||
check-wasm-embedded.sh buildfresh exempt binary embed check
|
||||
check-bootstrap-tag.sh repostate exempt VERSION stamp drift check
|
||||
check-cli-executable.sh repostate exempt file-mode check
|
||||
check-no-tracked-symlinks.sh repostate exempt git index state check
|
||||
|
@@ -46,6 +46,25 @@ set -uo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# W0 fix-wave (Tier-1 #16): PGLite schema snapshot, DEFAULT-ON for the plain
|
||||
# `bun run test` loop. 500+ test files each cold-boot PGLite + replay 126
|
||||
# migrations without it; the fixture was previously enabled ONLY inside
|
||||
# scripts/ci-local.sh, so the everyday loop paid the full cost. The build
|
||||
# script is idempotent (hash short-circuit) and concurrency-safe (mkdir
|
||||
# lock, D5.8), and its hash folds handler-migration source (D5.13), so an
|
||||
# unconditional call here is cheap and always current. Runs BEFORE the shard
|
||||
# fan-out — shards inherit a finished fixture. Opt out: GBRAIN_NO_SNAPSHOT=1
|
||||
# (the migration-replay canary tests clear the env themselves regardless).
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
if [ "${GBRAIN_NO_SNAPSHOT:-0}" != "1" ]; then
|
||||
if bun run build:pglite-snapshot >/dev/null 2>&1; then
|
||||
export GBRAIN_PGLITE_SNAPSHOT=test/fixtures/pglite-snapshot.tar
|
||||
else
|
||||
echo "[run-unit-parallel] snapshot build failed (non-fatal) — tests run with cold init" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# CPU detection: Apple Silicon perf cores → Mac total physical → nproc → 4.
|
||||
# Returns a single positive integer.
|
||||
|
||||
@@ -72,6 +72,14 @@ CHECKS=(
|
||||
"check:bootstrap-tag"
|
||||
"check:bootstrap-templates"
|
||||
"check:skill-refs"
|
||||
# W0 fix-wave (Tier-1 #11): guard self-tests — every scanner guard proves it
|
||||
# can fail (bad fixture → exit 1) before it counts as coverage. Registry:
|
||||
# scripts/guards-manifest.tsv (package.json's stale `check:all` copy deleted).
|
||||
"check:guard-self-test"
|
||||
# Previously reachable ONLY from the deleted check:all (i.e. never run):
|
||||
"check:newlines"
|
||||
"check:exports-count"
|
||||
"check:no-legacy-getconnection"
|
||||
"typecheck"
|
||||
)
|
||||
|
||||
|
||||
+14
-4
@@ -2352,16 +2352,26 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
try {
|
||||
switch (command) {
|
||||
case 'import': {
|
||||
const { runImport } = await import('./commands/import.ts');
|
||||
const { runImport, ImportAbortError } = await import('./commands/import.ts');
|
||||
// v0.41 (Codex r2 #3 fix): honor errors counter for exit code.
|
||||
// runImport's per-file catch already records failures, but the
|
||||
// CLI was discarding the result so the process exited 0 even
|
||||
// when files failed (e.g. content-sanity hard-block throws,
|
||||
// size-cap throws, parse errors). Surface non-zero on errors > 0
|
||||
// so wrappers (sync, CI scripts, `&& gbrain doctor`) propagate.
|
||||
const importResult = await runImport(engine, args);
|
||||
if (importResult.errors > 0) {
|
||||
setCliExitVerdict(1);
|
||||
try {
|
||||
const importResult = await runImport(engine, args);
|
||||
if (importResult.errors > 0) {
|
||||
setCliExitVerdict(1);
|
||||
}
|
||||
} catch (e) {
|
||||
// W0 (Tier-1 #5): runImport throws typed aborts instead of
|
||||
// process.exit(1) so in-process callers (sync_brain MCP op,
|
||||
// autopilot, minion handler) survive a preflight failure. The CLI
|
||||
// keeps the exact pre-fix behavior: message already printed at the
|
||||
// throw site, exit non-zero here.
|
||||
if (e instanceof ImportAbortError) process.exit(e.exitCode);
|
||||
throw e;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
+13
-14
@@ -1,6 +1,7 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { embedBatch, currentEmbeddingSignature } from '../core/embedding.ts';
|
||||
import type { ChunkInput } from '../core/types.ts';
|
||||
import { carryChunkMetadata } from '../core/embed-stale.ts';
|
||||
import { chunkText } from '../core/chunkers/recursive.ts';
|
||||
import { createProgress, type ProgressReporter } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
@@ -784,9 +785,11 @@ async function embedPage(
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry code-chunk metadata (language, symbol_name, symbol_type, line range,
|
||||
* parent scope, doc comment, qualified name) from a loaded Chunk back into a
|
||||
* ChunkInput destined for upsertChunks.
|
||||
* Carry per-chunk metadata — modality (the W0 fix: its omission flipped
|
||||
* image chunks to text) plus the code fields (language, symbol_name,
|
||||
* symbol_type, line range, parent scope, doc comment, qualified name) — from
|
||||
* a loaded Chunk back into a ChunkInput destined for upsertChunks. The
|
||||
* shared carryChunkMetadata list (core/embed-stale.ts) is authoritative.
|
||||
*
|
||||
* Issue #769: every re-embed used to strip these fields, and upsertChunks
|
||||
* overwrites (does not COALESCE) the metadata columns from EXCLUDED, so
|
||||
@@ -795,17 +798,13 @@ async function embedPage(
|
||||
* (embedPage, embedAll non-stale, embedAllStale) in lock-step.
|
||||
*/
|
||||
function preserveCodeMetadata(loaded: any, base: ChunkInput): ChunkInput {
|
||||
return {
|
||||
...base,
|
||||
language: loaded.language ?? undefined,
|
||||
symbol_name: loaded.symbol_name ?? undefined,
|
||||
symbol_type: loaded.symbol_type ?? undefined,
|
||||
start_line: loaded.start_line ?? undefined,
|
||||
end_line: loaded.end_line ?? undefined,
|
||||
parent_symbol_path: loaded.parent_symbol_path ?? undefined,
|
||||
doc_comment: loaded.doc_comment ?? undefined,
|
||||
symbol_name_qualified: loaded.symbol_name_qualified ?? undefined,
|
||||
};
|
||||
// W0 fix-wave (Tier-1 #3, CONFIRMED): delegate to the single shared carry
|
||||
// list. This local copy was missing `modality`, so every CLI re-embed path
|
||||
// (embedPage, embedAll, embedAllStale) flipped image chunks to
|
||||
// modality='text' — upsertChunks overwrites from EXCLUDED — silently
|
||||
// zeroing image retrieval until the next full import. The minion twin in
|
||||
// core/embed-stale.ts carried it correctly; one list now serves both.
|
||||
return carryChunkMetadata(loaded, base);
|
||||
}
|
||||
|
||||
async function embedAll(
|
||||
|
||||
+26
-5
@@ -64,6 +64,27 @@ function defaultWorkers(): number {
|
||||
return Math.min(byPool, byCpu, byMem);
|
||||
}
|
||||
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #5): typed abort for runImport's preflight/argv
|
||||
* failures. Pre-fix these five sites called process.exit(1) directly —
|
||||
* correct for the CLI, but runImport is ALSO invoked in-process by the
|
||||
* sync_brain MCP op (via performFullSync), the autopilot daemon, and the
|
||||
* minion sync handler, so a first sync with unconfigured embedding
|
||||
* credentials TERMINATED the MCP server / daemon / worker mid-call. The
|
||||
* user-facing messages are printed BEFORE the throw (byte-identical CLI
|
||||
* output); the CLI dispatch site maps this error back to exit(exitCode).
|
||||
*/
|
||||
export class ImportAbortError extends Error {
|
||||
readonly exitCode: number;
|
||||
/** True: the user-facing message was already printed at the throw site. */
|
||||
readonly alreadyReported = true;
|
||||
constructor(reason: string, exitCode = 1) {
|
||||
super(`import aborted: ${reason}`);
|
||||
this.name = 'ImportAbortError';
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
|
||||
/** Bug 9 — surface per-file failures so callers (performFullSync) can gate state advances. */
|
||||
export interface RunImportResult {
|
||||
imported: number;
|
||||
@@ -129,7 +150,7 @@ export async function runImport(
|
||||
} catch (e) {
|
||||
console.error(`\n${e instanceof Error ? e.message : e}`);
|
||||
console.error('Tip: run `gbrain import <dir> --no-embed` to import without embedding now.');
|
||||
process.exit(1);
|
||||
throw new ImportAbortError('embedding disabled (deferred-setup sentinel)');
|
||||
}
|
||||
|
||||
// v0.41.6.0 D1: preflight embedding credentials. Closes the bug class
|
||||
@@ -147,7 +168,7 @@ export async function runImport(
|
||||
console.error(e.userMessage);
|
||||
console.error('');
|
||||
}
|
||||
process.exit(1);
|
||||
throw new ImportAbortError('embedding credentials missing');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
@@ -221,7 +242,7 @@ export async function runImport(
|
||||
workerCount = parseWorkers(workersArg ?? undefined) ?? 1;
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
throw new ImportAbortError('invalid --workers value');
|
||||
}
|
||||
// Find dir: first non-flag arg that isn't a value for --workers
|
||||
const flagValues = new Set<number>();
|
||||
@@ -231,7 +252,7 @@ export async function runImport(
|
||||
|
||||
if (!dirArg) {
|
||||
console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--include-gitignored] [--json]');
|
||||
process.exit(1);
|
||||
throw new ImportAbortError('no import directory given');
|
||||
}
|
||||
// #1728: capture the import target ONCE as an absolute real path. Every
|
||||
// downstream consumer of `dir` (collection, checkpoint load/save, resume
|
||||
@@ -244,7 +265,7 @@ export async function runImport(
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error(`Import target is not readable: ${dirArg} (${msg})`);
|
||||
process.exit(1);
|
||||
throw new ImportAbortError(`import target not readable: ${dirArg}`);
|
||||
}
|
||||
|
||||
// v0.31.2: collect under the right strategy. Pre-fix this called
|
||||
|
||||
@@ -1937,10 +1937,11 @@ export async function registerBuiltinHandlers(
|
||||
});
|
||||
|
||||
worker.register('import', async (job) => {
|
||||
// import.ts Core extraction deferred to v0.12.0 (import has parallel
|
||||
// workers + checkpointing). Keep the CLI wrapper call but note the
|
||||
// worker-kill risk is bounded: import's only process.exit fires on
|
||||
// a missing dir arg, which this handler always passes.
|
||||
// import.ts Core extraction deferred (import has parallel workers +
|
||||
// checkpointing; the typed-API split lands in W7 of the fix-wave).
|
||||
// W0 (Tier-1 #5): runImport no longer contains ANY process.exit — all
|
||||
// five preflight sites throw typed ImportAbortError, which this
|
||||
// handler's catch converts to a normal failJob. No worker-kill risk.
|
||||
const { runImport } = await import('./import.ts');
|
||||
const importArgs: string[] = [];
|
||||
if (job.data.dir) importArgs.push(String(job.data.dir));
|
||||
|
||||
+52
-45
@@ -441,6 +441,24 @@ export interface LintOpts {
|
||||
* single-file targets.
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #14): per-page hook fired for every page WITH
|
||||
* issues, after this run's fix attempt for that page — fixedCount is the
|
||||
* number of fixes just applied (0 when --fix is off or nothing was
|
||||
* fixable). The CLI passes a printer so human detail and the aggregate
|
||||
* counts come from ONE scan — pre-fix, runLint ran its own full
|
||||
* read+lint+fix loop and THEN called runLintCore for the summary, linting
|
||||
* every page twice and reporting "0 auto-fixed" because the second pass
|
||||
* saw already-fixed files.
|
||||
*/
|
||||
onPageIssues?: (relPath: string, issues: LintIssue[], fixedCount: number) => void;
|
||||
/** Companion to onPageIssues: per-page progress tick (CLI progress bar). */
|
||||
onPageScanned?: () => void;
|
||||
/**
|
||||
* Fired once with the collected page count before scanning starts, so the
|
||||
* CLI can size its progress bar without walking the tree a second time.
|
||||
*/
|
||||
onPagesCollected?: (count: number) => void;
|
||||
}
|
||||
|
||||
export interface LintResult {
|
||||
@@ -468,6 +486,7 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
|
||||
const isSingleFile = statSync(opts.target).isFile();
|
||||
const pages = isSingleFile ? [opts.target] : collectPages(opts.target, opts.exclude ?? []);
|
||||
opts.onPagesCollected?.(pages.length);
|
||||
|
||||
// Resolve content-sanity config once for this lint run (D1: lift DB
|
||||
// config when reachable). Caller can pre-pass via opts.contentSanity
|
||||
@@ -490,21 +509,25 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
const content = readFileSync(page, 'utf-8');
|
||||
const issues = lintContent(content, isSingleFile ? page : relative(opts.target, page), lintOpts);
|
||||
const relPath = isSingleFile ? page : relative(opts.target, page);
|
||||
const issues = lintContent(content, relPath, lintOpts);
|
||||
opts.onPageScanned?.();
|
||||
if (issues.length === 0) continue;
|
||||
pagesWithIssues++;
|
||||
totalIssues += issues.length;
|
||||
|
||||
let fixCount = 0;
|
||||
if (opts.fix && issues.some(i => i.fixable)) {
|
||||
const fixed = fixContent(content);
|
||||
if (fixed !== content) {
|
||||
const fixCount = issues.filter(i => i.fixable).length;
|
||||
fixCount = issues.filter(i => i.fixable).length;
|
||||
totalFixed += fixCount;
|
||||
if (!opts.dryRun) {
|
||||
writeFileSync(page, fixed);
|
||||
}
|
||||
}
|
||||
}
|
||||
opts.onPageIssues?.(relPath, issues, fixCount);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -547,57 +570,41 @@ export async function runLint(args: string[]) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Single file or directory — print human detail as we go, then rely on
|
||||
// Core for the aggregate numbers at the end.
|
||||
const isSingleFile = statSync(target).isFile();
|
||||
const pages = isSingleFile ? [target] : collectPages(target, extraExcludes);
|
||||
|
||||
// W0 fix-wave (Tier-1 #14): ONE scan. Pre-fix this function ran its own
|
||||
// full read+lint+fix loop for human output and THEN called runLintCore for
|
||||
// the summary — every page linted twice, and with --fix the second pass
|
||||
// saw already-fixed files so the summary reported "0 auto-fixed" after
|
||||
// fixing N. Human detail now streams from runLintCore's per-page hooks
|
||||
// and the counts come from the same single pass.
|
||||
// Progress on stderr. Stdout keeps the per-issue human output it always had.
|
||||
// Ship-review perf catch: the tree is walked ONCE — runLintCore reports the
|
||||
// collected count via onPagesCollected (pre-fix the CLI ran its own
|
||||
// collectPages just to size the progress bar, a second full readdir/stat
|
||||
// walk on every directory lint).
|
||||
const { createProgress } = await import('../core/progress.ts');
|
||||
const { getCliOptions, cliOptsToProgressOptions } = await import('../core/cli-options.ts');
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
progress.start('lint.pages', pages.length);
|
||||
|
||||
// v0.41 (D1): resolve content-sanity config once for this lint run.
|
||||
// Mirrors runLintCore. The two paths must agree because runLint
|
||||
// prints human details inline; runLintCore at end computes the
|
||||
// aggregate. Sharing the resolved opts keeps both surfaces seeing
|
||||
// the same rule firings.
|
||||
const contentSanity = await resolveLintContentSanity();
|
||||
const lintContentOpts: LintContentOpts = { contentSanity };
|
||||
|
||||
for (const page of pages) {
|
||||
const content = readFileSync(page, 'utf-8');
|
||||
const relPath = isSingleFile ? page : relative(target, page);
|
||||
const issues = lintContent(content, relPath, lintContentOpts);
|
||||
progress.tick(1);
|
||||
if (issues.length === 0) continue;
|
||||
|
||||
console.log(`\n${relPath}:`);
|
||||
for (const issue of issues) {
|
||||
const fixLabel = issue.fixable ? ' [fixable]' : '';
|
||||
console.log(` L${issue.line} ${issue.rule}: ${issue.message}${fixLabel}`);
|
||||
}
|
||||
|
||||
if (doFix && issues.some(i => i.fixable)) {
|
||||
const fixed = fixContent(content);
|
||||
if (fixed !== content) {
|
||||
const fixCount = issues.filter(i => i.fixable).length;
|
||||
if (!dryRun) {
|
||||
writeFileSync(page, fixed);
|
||||
}
|
||||
console.log(` ${dryRun ? '(dry run) ' : ''}Fixed ${fixCount} issue(s)`);
|
||||
const result = await runLintCore({
|
||||
target,
|
||||
fix: doFix,
|
||||
dryRun,
|
||||
exclude: extraExcludes,
|
||||
onPagesCollected: (count) => progress.start('lint.pages', count),
|
||||
onPageScanned: () => progress.tick(1),
|
||||
onPageIssues: (relPath, issues, fixedCount) => {
|
||||
console.log(`\n${relPath}:`);
|
||||
for (const issue of issues) {
|
||||
const fixLabel = issue.fixable ? ' [fixable]' : '';
|
||||
console.log(` L${issue.line} ${issue.rule}: ${issue.message}${fixLabel}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (fixedCount > 0) {
|
||||
console.log(` ${dryRun ? '(dry run) ' : ''}Fixed ${fixedCount} issue(s)`);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
progress.finish();
|
||||
|
||||
// Re-run core for the aggregate counts (cheap; re-parses contents but
|
||||
// produces canonical numbers for the summary line).
|
||||
// Pass contentSanity through so runLintCore skips its own resolve
|
||||
// (we already resolved once for the human-detail loop above).
|
||||
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity, exclude: extraExcludes });
|
||||
console.log(`\n${result.pages_scanned} pages scanned. ${result.total_issues} issue(s) in ${result.pages_with_issues} page(s).`);
|
||||
if (doFix) {
|
||||
console.log(`${dryRun ? '(dry run) ' : ''}${result.total_fixed} auto-fixed.`);
|
||||
|
||||
@@ -101,9 +101,15 @@ function emitError(jsonOutput: boolean, code: string, message: string): void {
|
||||
}
|
||||
|
||||
async function promptYesNo(question: string): Promise<boolean> {
|
||||
// W0 fix-wave (Tier-1 #15): non-interactive stdin (CI, pipes, spawned
|
||||
// agents) must resolve to the safe default instead of hanging forever —
|
||||
// this prompt had no TTY guard and no close/EOF handler, so a piped or
|
||||
// closed stdin parked the process permanently.
|
||||
if (!process.stdin.isTTY) return false;
|
||||
// Prompt on stderr: stdout stays clean for --json payloads.
|
||||
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
||||
return new Promise((resolve) => {
|
||||
rl.on('close', () => resolve(false)); // EOF (^D) = decline, never hang
|
||||
rl.question(`${question} [y/N] `, (answer) => {
|
||||
rl.close();
|
||||
resolve(/^y(es)?$/i.test(answer.trim()));
|
||||
|
||||
@@ -331,18 +331,29 @@ function fail(jsonOutput: boolean, reason: string, message: string): never {
|
||||
}
|
||||
|
||||
async function promptYesNo(question: string): Promise<boolean> {
|
||||
// Minimal TTY prompt — no external deps. Bun's process.stdin reads
|
||||
// a single line synchronously via the async iterator.
|
||||
process.stdout.write(`${question} (y/N): `);
|
||||
// W0 fix-wave (Tier-1 #15): non-interactive stdin (CI, pipes, spawned
|
||||
// agents) resolves to the safe default instead of hanging — this prompt
|
||||
// had no TTY guard and no end/EOF path, so a closed stdin parked the
|
||||
// process permanently. This command wipes the store; decline-by-default
|
||||
// is the only safe non-interactive answer (--yes stays the escape hatch).
|
||||
if (!process.stdin.isTTY) return false;
|
||||
// Prompt on stderr so stdout stays clean for --json payloads.
|
||||
process.stderr.write(`${question} (y/N): `);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const stdin = process.stdin as any;
|
||||
stdin.setEncoding?.('utf8');
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const cleanup = () => {
|
||||
stdin.off?.('data', onData);
|
||||
stdin.off?.('end', onEnd);
|
||||
};
|
||||
const onEnd = () => { cleanup(); resolve(false); }; // EOF = decline
|
||||
const onData = (chunk: string) => {
|
||||
const answer = chunk.trim().toLowerCase();
|
||||
stdin.off?.('data', onData);
|
||||
cleanup();
|
||||
resolve(answer === 'y' || answer === 'yes');
|
||||
};
|
||||
stdin.on?.('data', onData);
|
||||
stdin.on?.('end', onEnd);
|
||||
});
|
||||
}
|
||||
|
||||
+264
-51
@@ -49,7 +49,7 @@ import { gbrainPath } from './config.ts';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { createProgress, type ProgressReporter } from './progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from './cli-options.ts';
|
||||
import { tryAcquireDbLock, reapDeadHolderLocks, type DbLockHandle } from './db-lock.ts';
|
||||
import { tryAcquireDbLock, reapDeadHolderLocks, LockStolenError, type DbLockHandle } from './db-lock.ts';
|
||||
import { assertValidSourceId } from './source-id.ts';
|
||||
|
||||
// ─── Types ─────────────────────────────────────────────────────────
|
||||
@@ -572,7 +572,14 @@ const getLockFilePathDefault = () => gbrainPath('cycle.lock');
|
||||
|
||||
export interface LockHandle {
|
||||
release: () => Promise<void>;
|
||||
refresh: () => Promise<void>;
|
||||
/**
|
||||
* W0 fix-wave: returns true while this holder still owns the lock. The
|
||||
* DB-backed handle runs a fenced UPDATE (db-lock.ts, D5.10) and returns
|
||||
* false after a steal; the file-lock handle rewrites its file and always
|
||||
* returns true (single-host, pid-checked at acquire). Callers that ignore
|
||||
* the boolean keep their old behavior.
|
||||
*/
|
||||
refresh: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -690,6 +697,7 @@ function acquireFileLock(lockPath = getLockFilePathDefault()): LockHandle | null
|
||||
} catch {
|
||||
/* non-fatal — a next-run stale check will notice */
|
||||
}
|
||||
return true;
|
||||
},
|
||||
release: async () => {
|
||||
try {
|
||||
@@ -726,21 +734,72 @@ function acquireFileLock(lockPath = getLockFilePathDefault()): LockHandle | null
|
||||
* Returns `undefined` when there's no lock AND no outer hook so phases
|
||||
* short-circuit via their `if (!opts.yieldDuringPhase) return;` guard.
|
||||
*/
|
||||
/**
|
||||
* W0 fix-wave: combine abort signals (external caller signal + the internal
|
||||
* lock-steal controller) into one REAL AbortSignal.
|
||||
*
|
||||
* Deliberately NOT AbortSignal.any: CycleOpts.signal has always been duck-
|
||||
* typed in practice (test stubs pass `{ aborted: false }` and flip the flag;
|
||||
* pre-W0 the raw object flowed straight into checkAborted, which only reads
|
||||
* `.aborted`/`.reason`). AbortSignal.any throws ERR_INVALID_ARG_TYPE on
|
||||
* those. Manual fan-in: real signals propagate via listener; listener-less
|
||||
* stubs are polled at 50ms — semantically the flip is seen within a tick,
|
||||
* and the RETURNED signal is a genuine AbortSignal so phases can hand it to
|
||||
* fetch/timers safely.
|
||||
*
|
||||
* ALWAYS call dispose() when the consuming scope ends (runCycle's finally):
|
||||
* the forward listeners live on the CALLER's signals, and long-lived callers
|
||||
* (the autopilot daemon passes its daemon-lifetime shutdown signal into
|
||||
* every cycle tick) would otherwise accumulate one listener + captured
|
||||
* controller per invocation forever (ship-review perf catch).
|
||||
*/
|
||||
export function anyAbortSignal(signals: AbortSignal[]): { signal: AbortSignal; dispose: () => void } {
|
||||
const c = new AbortController();
|
||||
const cleanups: Array<() => void> = [];
|
||||
const forward = (s: AbortSignal) => { if (!c.signal.aborted) c.abort(s.reason); };
|
||||
for (const s of signals) {
|
||||
if (!s) continue;
|
||||
if (s.aborted) { forward(s); break; }
|
||||
if (typeof (s as Partial<AbortSignal>).addEventListener === 'function') {
|
||||
const listener = () => forward(s);
|
||||
s.addEventListener('abort', listener, { once: true });
|
||||
cleanups.push(() => s.removeEventListener('abort', listener));
|
||||
} else {
|
||||
const t = setInterval(() => { if (s.aborted) { forward(s); clearInterval(t); } }, 50);
|
||||
(t as unknown as { unref?: () => void }).unref?.();
|
||||
c.signal.addEventListener('abort', () => clearInterval(t), { once: true });
|
||||
cleanups.push(() => clearInterval(t));
|
||||
}
|
||||
}
|
||||
return {
|
||||
signal: c.signal,
|
||||
dispose: () => { for (const fn of cleanups) { try { fn(); } catch { /* best effort */ } } },
|
||||
};
|
||||
}
|
||||
|
||||
export function buildYieldDuringPhase(
|
||||
lock: LockHandle | null,
|
||||
outer?: () => Promise<void>,
|
||||
onStolen?: (err: LockStolenError) => void,
|
||||
): (() => Promise<void>) | undefined {
|
||||
if (!lock && !outer) return undefined;
|
||||
return async () => {
|
||||
if (lock) {
|
||||
try {
|
||||
await lock.refresh();
|
||||
const stillOwned = await lock.refresh();
|
||||
if (stillOwned === false) {
|
||||
// W0 (D5.10/D5.11): the fenced refresh proved the lock is gone.
|
||||
// Don't throw mid-LLM-call — signal the cycle's steal controller
|
||||
// so the run stops at the next boundary / raced await instead of
|
||||
// compounding writes against a concurrent successor.
|
||||
console.error('[cycle] lock refresh matched 0 rows — lock stolen; signaling cycle abort');
|
||||
onStolen?.(new LockStolenError('cycle-lock'));
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
// Non-fatal: a refresh error doesn't crash the phase. If the
|
||||
// lock truly expired and was stolen, the next acquire by another
|
||||
// worker has already happened — let this run wind down rather
|
||||
// than throw mid-phase.
|
||||
// Non-fatal: a TRANSIENT refresh error doesn't crash the phase (it
|
||||
// is not evidence of a steal; the TTL is the backstop and the next
|
||||
// tick retries).
|
||||
console.error(`[cycle] lock refresh failed (non-fatal): ${msg}`);
|
||||
}
|
||||
}
|
||||
@@ -750,6 +809,68 @@ export function buildYieldDuringPhase(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #1 + D5.11): runCycle-owned serialized lock refresher.
|
||||
*
|
||||
* Production callers never set `CycleOpts.yieldDuringPhase`, so before this
|
||||
* timer the cycle lock was refreshed only by phases that happened to receive
|
||||
* a wrapped hook — with a 5-min TTL against 35-min subagent waits, the lock
|
||||
* was effectively NEVER refreshed in production (verified in the 2026-08-14
|
||||
* audit). This interval owns the CYCLE lock only; Minion job-lock renewal
|
||||
* stays on the yieldDuringPhase/yieldBetweenPhases hooks (the cycle.ts:618
|
||||
* decision — a background timer must not replace the phase-boundary hook).
|
||||
*
|
||||
* Serialized: at most one refresh in flight (a slow refresh never overlaps
|
||||
* the next tick). On a fenced refresh returning false, aborts `controller`
|
||||
* with a LockStolenError; a thrown (transient) refresh error is logged and
|
||||
* retried next tick — the TTL is the backstop.
|
||||
*/
|
||||
export function startCycleLockRefresher(
|
||||
lock: LockHandle,
|
||||
controller: AbortController,
|
||||
lockId: string,
|
||||
intervalMs: number = resolveCycleLockRefreshMs(),
|
||||
): () => void {
|
||||
let inFlight = false;
|
||||
const timer = setInterval(() => {
|
||||
if (inFlight || controller.signal.aborted) return;
|
||||
inFlight = true;
|
||||
void (async () => {
|
||||
try {
|
||||
const stillOwned = await lock.refresh();
|
||||
if (stillOwned === false && !controller.signal.aborted) {
|
||||
controller.abort(new LockStolenError(lockId));
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[cycle] background lock refresh failed (non-fatal, retrying next tick): ${msg}`);
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
})();
|
||||
}, intervalMs);
|
||||
// Don't let the refresher pin the event loop open past real work.
|
||||
(timer as unknown as { unref?: () => void }).unref?.();
|
||||
return () => clearInterval(timer);
|
||||
}
|
||||
|
||||
/** Refresh 6x per TTL window (~50s at the 5-min TTL), matching withRefreshingLock's cadence. */
|
||||
const CYCLE_LOCK_REFRESH_INTERVAL_MS = Math.max(15_000, LOCK_TTL_MS / 6);
|
||||
|
||||
/**
|
||||
* GBRAIN_CYCLE_LOCK_REFRESH_MS: env-only escape hatch (incident tuning +
|
||||
* deterministic tests), same posture as the GBRAIN_SYNC_* knobs. Floor of
|
||||
* 10ms guards against a zero/NaN wedging the event loop.
|
||||
*/
|
||||
function resolveCycleLockRefreshMs(): number {
|
||||
const raw = process.env.GBRAIN_CYCLE_LOCK_REFRESH_MS;
|
||||
if (raw) {
|
||||
const n = Number(raw);
|
||||
if (Number.isFinite(n) && n >= 10) return n;
|
||||
}
|
||||
return CYCLE_LOCK_REFRESH_INTERVAL_MS;
|
||||
}
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────
|
||||
|
||||
function makeErrorFromException(e: unknown, fallbackClass = 'InternalError'): PhaseError {
|
||||
@@ -1712,8 +1833,18 @@ export async function runCycle(
|
||||
lock = pgliteFileLock
|
||||
? {
|
||||
refresh: async () => {
|
||||
await dbLock!.refresh();
|
||||
await pgliteFileLock!.refresh();
|
||||
// W0 (D5.10): the DB row is the authoritative multi-writer
|
||||
// identity; the file refresh is best-effort freshness. Propagate
|
||||
// the fenced result so steal detection reaches the refresher.
|
||||
// Red-team catch: NEVER rewrite the file half after the fence
|
||||
// reports loss — the unconditional rewrite let the losing
|
||||
// holder clobber the successor's file lock (our pid back in the
|
||||
// file) on the very tick it detected the steal, after which our
|
||||
// pid-checked file release would DELETE the successor's only
|
||||
// host-local protection mid-run.
|
||||
const stillOwned = await dbLock!.refresh();
|
||||
if (stillOwned) await pgliteFileLock!.refresh();
|
||||
return stillOwned;
|
||||
},
|
||||
release: async () => {
|
||||
try {
|
||||
@@ -1745,6 +1876,51 @@ export async function runCycle(
|
||||
}
|
||||
}
|
||||
|
||||
// W0 fix-wave (Tier-1 #1 + D5.11): lock-steal detection and propagation.
|
||||
//
|
||||
// refresher (50s tick) ──fenced UPDATE──▶ 0 rows? ──▶ stolen.abort(LockStolenError)
|
||||
// │ │
|
||||
// └── yieldDuringPhase hooks also report steals ────────┤
|
||||
// ▼
|
||||
// cycleSignal = any(opts.signal, stolen.signal) → checkAborted() at every
|
||||
// phase boundary; raceStolen() additionally races the 5 long-phase awaits
|
||||
// (their opts can't carry a signal yet — full threading lands in W6).
|
||||
//
|
||||
// External aborts (opts.signal) keep today's throw-out semantics; ONLY a
|
||||
// steal is caught below and returned as a structured partial report.
|
||||
const externalSignal = opts.signal;
|
||||
const stolen: AbortController | null = lock ? new AbortController() : null;
|
||||
const combinedSignal = stolen && externalSignal
|
||||
? anyAbortSignal([externalSignal, stolen.signal])
|
||||
: null;
|
||||
const cycleSignal: AbortSignal | undefined = combinedSignal
|
||||
? combinedSignal.signal
|
||||
: (stolen?.signal ?? externalSignal);
|
||||
const stopRefresher: (() => void) | undefined = lock && stolen
|
||||
? startCycleLockRefresher(lock, stolen, cycleLockIdFor(opts.sourceId))
|
||||
: undefined;
|
||||
const onStolen = stolen ? (e: LockStolenError) => { if (!stolen.signal.aborted) stolen.abort(e); } : undefined;
|
||||
const raceStolen = !stolen
|
||||
? <T,>(p: Promise<T>): Promise<T> => p
|
||||
: <T,>(p: Promise<T>): Promise<T> => {
|
||||
if (stolen.signal.aborted) return Promise.reject(stolen.signal.reason);
|
||||
let onAbort!: () => void;
|
||||
const abortP = new Promise<never>((_, rej) => {
|
||||
onAbort = () => rej(stolen.signal.reason);
|
||||
stolen.signal.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
return Promise.race([p, abortP]).finally(() => {
|
||||
stolen.signal.removeEventListener('abort', onAbort);
|
||||
}) as Promise<T>;
|
||||
};
|
||||
let lockStolenAbort = false;
|
||||
// Raced variant for the 5 long phases (synthesize / extract_atoms / patterns
|
||||
// / synthesize_concepts / consolidate): their opts can't carry a signal yet
|
||||
// (W6), so a steal must be able to stop the WAIT even though the phase's
|
||||
// in-flight work runs to its own bounded timeout. Steal-free cycles behave
|
||||
// byte-identically to timePhase.
|
||||
const racedTimePhase = <T,>(fn: () => Promise<T>) => raceStolen(timePhase(fn));
|
||||
|
||||
// #1972: reap dead-holder sync/cycle locks at cycle start — before the sync
|
||||
// phase needs them — so a crashed sync's stranded lock self-heals THIS tick
|
||||
// instead of waiting out its TTL. Best-effort, namespace-scoped + host-scoped;
|
||||
@@ -1766,12 +1942,12 @@ export async function runCycle(
|
||||
try {
|
||||
// ── Phase 1: lint ────────────────────────────────────────────
|
||||
if (phases.includes('lint')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (brainDir === null) {
|
||||
phaseResults.push(skipNoBrainDir('lint'));
|
||||
} else {
|
||||
progress.start('cycle.lint');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseLint(brainDir, dryRun, engine, opts.signal));
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseLint(brainDir, dryRun, engine, cycleSignal));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
@@ -1781,7 +1957,7 @@ export async function runCycle(
|
||||
|
||||
// ── Phase 2: backlinks ──────────────────────────────────────
|
||||
if (phases.includes('backlinks')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (brainDir === null) {
|
||||
phaseResults.push(skipNoBrainDir('backlinks'));
|
||||
} else {
|
||||
@@ -1806,7 +1982,7 @@ export async function runCycle(
|
||||
let syncAttempted = false;
|
||||
let synthesizeWrittenSlugs: string[] | undefined;
|
||||
if (phases.includes('sync')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'sync',
|
||||
@@ -1820,7 +1996,13 @@ export async function runCycle(
|
||||
} else {
|
||||
progress.start('cycle.sync');
|
||||
syncAttempted = true; // sync ran its work; undefined pagesAffected now means failure
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseSync(engine, brainDir, dryRun, pull, phases.includes('extract')));
|
||||
// Red-team catch: sync is production's LONGEST phase (resumable
|
||||
// imports can run hours) and was the one long await outside steal
|
||||
// coverage. Raced like the other five: an abandoned wait is safe —
|
||||
// sync checkpoints its progress, holds its own per-source lock (the
|
||||
// successor's sync phase skips with lock-busy), and its stall
|
||||
// watchdog bounds the dangling import. Signal threading lands in W6.
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhaseSync(engine, brainDir, dryRun, pull, phases.includes('extract')));
|
||||
result.duration_ms = duration_ms;
|
||||
// Capture changed slugs for incremental extract.
|
||||
syncPagesAffected = (result as SyncPhaseResult).pagesAffected;
|
||||
@@ -1845,10 +2027,13 @@ export async function runCycle(
|
||||
} else {
|
||||
progress.start('cycle.synthesize');
|
||||
const { runPhaseSynthesize } = await import('./cycle/synthesize.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseSynthesize(engine, {
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhaseSynthesize(engine, {
|
||||
brainDir,
|
||||
dryRun,
|
||||
yieldDuringPhase: opts.yieldDuringPhase,
|
||||
// W0 (Tier-1 #1): wrap the caller hook so this phase ALSO refreshes
|
||||
// the cycle lock (pre-fix these sites passed the raw — in production
|
||||
// always-undefined — hook, so long phases never refreshed).
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase, onStolen),
|
||||
inputFile: opts.synthInputFile,
|
||||
date: opts.synthDate,
|
||||
from: opts.synthFrom,
|
||||
@@ -1873,7 +2058,7 @@ export async function runCycle(
|
||||
|
||||
// ── Phase 5: extract (now picks up synthesize output) ───────
|
||||
if (phases.includes('extract')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'extract',
|
||||
@@ -1889,7 +2074,7 @@ export async function runCycle(
|
||||
// If sync didn't run (phases exclude it) or failed, syncPagesAffected
|
||||
// is undefined → extract falls back to full walk (safe default).
|
||||
progress.start('cycle.extract');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseExtract(engine, brainDir, dryRun, syncPagesAffected, opts.signal, cycleSourceId));
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseExtract(engine, brainDir, dryRun, syncPagesAffected, cycleSignal, cycleSourceId));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
@@ -1905,7 +2090,7 @@ export async function runCycle(
|
||||
// refuses to run while v0.31 legacy facts are pending the
|
||||
// v0_32_2 backfill (Codex R2-#7).
|
||||
if (phases.includes('extract_facts')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'extract_facts',
|
||||
@@ -1938,7 +2123,7 @@ export async function runCycle(
|
||||
const syncRanButFailed = syncAttempted && syncPagesAffected === undefined;
|
||||
const xfSlugs = syncRanButFailed ? [] : syncPagesAffected;
|
||||
const { result, duration_ms } = await timePhase(() =>
|
||||
runPhaseExtractFacts(engine, brainDir, xfSourceId, dryRun, xfSlugs, opts.signal));
|
||||
runPhaseExtractFacts(engine, brainDir, xfSourceId, dryRun, xfSlugs, cycleSignal));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
@@ -1959,7 +2144,7 @@ export async function runCycle(
|
||||
// resolved active pack's `phases:` list ONLY; not the extends chain
|
||||
// or borrow_from targets.
|
||||
if (phases.includes('extract_atoms')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'extract_atoms',
|
||||
@@ -1995,13 +2180,13 @@ export async function runCycle(
|
||||
...(synthesizeWrittenSlugs ?? []),
|
||||
]
|
||||
: undefined;
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseExtractAtoms(engine, {
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhaseExtractAtoms(engine, {
|
||||
brainDir: brainDir ?? undefined,
|
||||
sourceId: xaSourceId,
|
||||
dryRun,
|
||||
affectedSlugs: xaAffectedSlugs,
|
||||
// v0.41.19.0 (T3): closure refreshes cycle lock + fires outer hook.
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase),
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase, onStolen),
|
||||
// v0.41.19.0 (T4): pass same reporter (not a child — cycle.ts
|
||||
// owns start/finish; phase only ticks).
|
||||
progress,
|
||||
@@ -2019,7 +2204,7 @@ export async function runCycle(
|
||||
// BATCH_SIZE * 10 chunks per invocation so a 60s watchdog tick stays
|
||||
// responsive even on a 100K-chunk brain.
|
||||
if (phases.includes('resolve_symbol_edges')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'resolve_symbol_edges',
|
||||
@@ -2057,10 +2242,13 @@ export async function runCycle(
|
||||
} else {
|
||||
progress.start('cycle.patterns');
|
||||
const { runPhasePatterns } = await import('./cycle/patterns.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhasePatterns(engine, {
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhasePatterns(engine, {
|
||||
brainDir,
|
||||
dryRun,
|
||||
yieldDuringPhase: opts.yieldDuringPhase,
|
||||
// W0 (Tier-1 #1): wrap the caller hook so this phase ALSO refreshes
|
||||
// the cycle lock (pre-fix these sites passed the raw — in production
|
||||
// always-undefined — hook, so long phases never refreshed).
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase, onStolen),
|
||||
once: opts.onceForPhase === 'patterns',
|
||||
deadlineAtMs: opts.deadlineAtMs ?? null,
|
||||
// #1586: scope pattern writes to the cycle's resolved source, same as
|
||||
@@ -2083,7 +2271,7 @@ export async function runCycle(
|
||||
// resolved active pack manifest; no-op when this phase isn't
|
||||
// declared. Real body in T6 — synthesize-concepts.ts is a stub today.
|
||||
if (phases.includes('synthesize_concepts')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'synthesize_concepts',
|
||||
@@ -2107,11 +2295,11 @@ export async function runCycle(
|
||||
} else {
|
||||
progress.start('cycle.synthesize_concepts');
|
||||
const { runPhaseSynthesizeConcepts } = await import('./cycle/synthesize-concepts.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseSynthesizeConcepts(engine, {
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhaseSynthesizeConcepts(engine, {
|
||||
brainDir: brainDir ?? undefined,
|
||||
dryRun,
|
||||
// v0.41.19.0 (T3): closure refreshes cycle lock + fires outer hook.
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase),
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase, onStolen),
|
||||
// v0.41.19.0 (T4): pass same reporter (not a child).
|
||||
progress,
|
||||
}));
|
||||
@@ -2127,7 +2315,7 @@ export async function runCycle(
|
||||
// every page touched in this cycle. Incremental mode uses union(sync,
|
||||
// synthesize); full mode walks every page in the brain.
|
||||
if (phases.includes('recompute_emotional_weight')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'recompute_emotional_weight',
|
||||
@@ -2167,7 +2355,7 @@ export async function runCycle(
|
||||
// per cluster, INSERT into takes(kind='fact'), mark facts as
|
||||
// consolidated_into. Never DELETE — facts are the audit trail.
|
||||
if (phases.includes('consolidate')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'consolidate',
|
||||
@@ -2179,10 +2367,13 @@ export async function runCycle(
|
||||
} else {
|
||||
progress.start('cycle.consolidate');
|
||||
const { runPhaseConsolidate } = await import('./cycle/phases/consolidate.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseConsolidate(engine, {
|
||||
const { result, duration_ms } = await racedTimePhase(() => runPhaseConsolidate(engine, {
|
||||
dryRun,
|
||||
yieldDuringPhase: opts.yieldDuringPhase,
|
||||
signal: opts.signal,
|
||||
// W0 (Tier-1 #1): wrap the caller hook so this phase ALSO refreshes
|
||||
// the cycle lock (pre-fix these sites passed the raw — in production
|
||||
// always-undefined — hook, so long phases never refreshed).
|
||||
yieldDuringPhase: buildYieldDuringPhase(lock, opts.yieldDuringPhase, onStolen),
|
||||
signal: cycleSignal,
|
||||
}));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
@@ -2218,7 +2409,7 @@ export async function runCycle(
|
||||
} as never;
|
||||
|
||||
if (phases.includes('propose_takes')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
progress.start('cycle.propose_takes');
|
||||
const { runPhaseProposeTakes } = await import('./cycle/propose-takes.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseProposeTakes(calibrationCtx, { repoPath: brainDir ?? undefined }) as Promise<PhaseResult>);
|
||||
@@ -2229,7 +2420,7 @@ export async function runCycle(
|
||||
}
|
||||
|
||||
if (phases.includes('grade_takes')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
progress.start('cycle.grade_takes');
|
||||
const { runPhaseGradeTakes } = await import('./cycle/grade-takes.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseGradeTakes(calibrationCtx, {}) as Promise<PhaseResult>);
|
||||
@@ -2240,7 +2431,7 @@ export async function runCycle(
|
||||
}
|
||||
|
||||
if (phases.includes('calibration_profile')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
progress.start('cycle.calibration_profile');
|
||||
const { runPhaseCalibrationProfile } = await import('./cycle/calibration-profile.ts');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseCalibrationProfile(calibrationCtx, {}) as Promise<PhaseResult>);
|
||||
@@ -2270,7 +2461,7 @@ export async function runCycle(
|
||||
// reports/drift-<date> page, mutates no takes regardless of
|
||||
// dream.drift.auto_update.
|
||||
if (phases.includes('drift')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'drift',
|
||||
@@ -2315,7 +2506,7 @@ export async function runCycle(
|
||||
// tracker passed in from the phase wrapper (NOT nested-wrapped in
|
||||
// core — would REPLACE not stack).
|
||||
if (phases.includes('conversation_facts_backfill')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'conversation_facts_backfill',
|
||||
@@ -2330,7 +2521,7 @@ export async function runCycle(
|
||||
const { result, duration_ms } = await timePhase(() =>
|
||||
runPhaseConversationFactsBackfill(engine, {
|
||||
dryRun,
|
||||
signal: opts.signal,
|
||||
signal: cycleSignal,
|
||||
once: opts.onceForPhase === 'conversation_facts_backfill',
|
||||
}),
|
||||
);
|
||||
@@ -2347,7 +2538,7 @@ export async function runCycle(
|
||||
// cost AND walltime caps; budget tracker created in the phase wrapper and
|
||||
// passed into the core (NOT nested-wrapped — would REPLACE not stack).
|
||||
if (phases.includes('enrich_thin')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'enrich_thin',
|
||||
@@ -2362,7 +2553,7 @@ export async function runCycle(
|
||||
const { result, duration_ms } = await timePhase(() =>
|
||||
runPhaseEnrichThin(engine, {
|
||||
dryRun,
|
||||
signal: opts.signal,
|
||||
signal: cycleSignal,
|
||||
once: opts.onceForPhase === 'enrich_thin',
|
||||
}),
|
||||
);
|
||||
@@ -2379,7 +2570,7 @@ export async function runCycle(
|
||||
// safety (D16): the phase ALWAYS runs in --no-mutate mode — proposed
|
||||
// bests land at skills/<name>/skillopt/best.md for review.
|
||||
if (phases.includes('skillopt')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'skillopt' as never,
|
||||
@@ -2396,7 +2587,7 @@ export async function runCycle(
|
||||
engine,
|
||||
dryRun,
|
||||
once: opts.onceForPhase === 'skillopt',
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
...(cycleSignal ? { signal: cycleSignal } : {}),
|
||||
}),
|
||||
);
|
||||
result.duration_ms = duration_ms;
|
||||
@@ -2408,7 +2599,7 @@ export async function runCycle(
|
||||
|
||||
// ── Phase 8: embed ──────────────────────────────────────────
|
||||
if (phases.includes('embed')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'embed',
|
||||
@@ -2419,7 +2610,7 @@ export async function runCycle(
|
||||
});
|
||||
} else {
|
||||
progress.start('cycle.embed');
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseEmbed(engine, dryRun, opts.signal));
|
||||
const { result, duration_ms } = await timePhase(() => runPhaseEmbed(engine, dryRun, cycleSignal));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
@@ -2429,7 +2620,7 @@ export async function runCycle(
|
||||
|
||||
// ── Phase 9: orphans ────────────────────────────────────────
|
||||
if (phases.includes('orphans')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'orphans',
|
||||
@@ -2455,7 +2646,7 @@ export async function runCycle(
|
||||
// (T15) and the disk-derived candidate set surfaced by `gbrain schema
|
||||
// review-candidates`.
|
||||
if (phases.includes('schema-suggest')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'schema-suggest',
|
||||
@@ -2499,7 +2690,7 @@ export async function runCycle(
|
||||
// 72h recovery window. Runs last so the rest of the cycle sees the
|
||||
// recoverable set; the purge then drops what's truly expired.
|
||||
if (phases.includes('purge')) {
|
||||
checkAborted(opts.signal);
|
||||
checkAborted(cycleSignal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'purge',
|
||||
@@ -2517,9 +2708,31 @@ export async function runCycle(
|
||||
}
|
||||
await safeYield(opts.yieldBetweenPhases);
|
||||
}
|
||||
} catch (e) {
|
||||
// W0 (Tier-1 #1): a lock steal aborts the run at the next boundary/raced
|
||||
// await. Completed phases' DB writes are durable (persisted-and-resumable
|
||||
// per D5.6); report a structured partial instead of throwing so daemon
|
||||
// callers (jobs.ts / autopilot) don't have to classify an exception.
|
||||
// External aborts (cycleSignal) keep the existing throw-out contract.
|
||||
const stolenFired = stolen?.signal.aborted === true
|
||||
&& stolen.signal.reason instanceof LockStolenError
|
||||
&& externalSignal?.aborted !== true;
|
||||
if (stolenFired) {
|
||||
lockStolenAbort = true;
|
||||
console.error(`[cycle] aborting: ${stolen!.signal.reason.message} — ${phaseResults.length} phase(s) completed before the steal; their writes are durable`);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
} finally {
|
||||
stopRefresher?.();
|
||||
// Detach the combined-signal forwarders from the CALLER's signal — the
|
||||
// autopilot daemon reuses one shutdown signal across every tick, and
|
||||
// undisposed listeners accumulate for the daemon's lifetime.
|
||||
combinedSignal?.dispose();
|
||||
if (lock) {
|
||||
try {
|
||||
// Safe after a steal: release() is fenced on (id, pid, acquired_at),
|
||||
// so it can never delete the successor's row (deletes 0 rows).
|
||||
await lock.release();
|
||||
} catch (e) {
|
||||
// #1470: best-effort, but never silent — a swallowed release failure
|
||||
@@ -2541,7 +2754,7 @@ export async function runCycle(
|
||||
// a cancelled run as a completed full cycle, which makes the next tick skip
|
||||
// work it never actually did. Treat an aborted signal as a non-success run:
|
||||
// skip the freshness stamp and report status 'partial' with reason 'aborted'.
|
||||
const aborted = opts.signal?.aborted === true;
|
||||
const aborted = cycleSignal?.aborted === true;
|
||||
|
||||
// #1972 (Decision 7A gating): attribute force-evicts. The minion worker
|
||||
// force-evicts a job 30s after abort and logs "handler ignored abort signal";
|
||||
@@ -2614,7 +2827,7 @@ export async function runCycle(
|
||||
timestamp,
|
||||
duration_ms,
|
||||
status: effectiveStatus,
|
||||
...(aborted ? { reason: 'aborted' } : stampWriteFailed ? { reason: 'stamp_write_failed' } : {}),
|
||||
...(lockStolenAbort ? { reason: 'lock_stolen' } : aborted ? { reason: 'aborted' } : stampWriteFailed ? { reason: 'stamp_write_failed' } : {}),
|
||||
...(stampWriteFailed ? { stamp_write_failed: stampWriteFailed } : {}),
|
||||
...(reapedLocks ? { reaped_dead_holder_locks: reapedLocks } : {}),
|
||||
brain_dir: opts.brainDir,
|
||||
|
||||
+77
-17
@@ -26,8 +26,38 @@ import type { BrainEngine } from './engine.ts';
|
||||
|
||||
export interface DbLockHandle {
|
||||
id: string;
|
||||
/**
|
||||
* Per-acquisition fencing identity (W0 fix-wave, D5.10): the row's
|
||||
* acquired_at rendered as epoch-seconds text (GUC-independent — timestamptz::text
|
||||
* would vary with per-session TimeZone/DateStyle across pools), captured at
|
||||
* acquire time. refresh()
|
||||
* and release() require an exact match, so a PID-reuse impostor (or this
|
||||
* handle after a steal) can never refresh or delete a successor's row.
|
||||
* `(id, holder_pid)` alone is NOT identity — PIDs recycle.
|
||||
*/
|
||||
acquiredAt: string;
|
||||
release: () => Promise<void>;
|
||||
refresh: () => Promise<void>;
|
||||
/**
|
||||
* Bump ttl_expires_at + last_refreshed_at. Returns true when this handle
|
||||
* still owns the row (exactly one row matched the fenced predicate);
|
||||
* false means the lock was stolen or released — the caller must stop
|
||||
* relying on mutual exclusion. Transient DB errors still THROW (they are
|
||||
* not evidence of a steal; the TTL is the backstop).
|
||||
*/
|
||||
refresh: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
* W0 fix-wave: thrown (or used as an AbortSignal reason) when a fenced
|
||||
* refresh discovers the lock row no longer belongs to this acquisition.
|
||||
*/
|
||||
export class LockStolenError extends Error {
|
||||
readonly lockId: string;
|
||||
constructor(lockId: string) {
|
||||
super(`lock '${lockId}' was stolen or released out from under this holder (fenced refresh matched 0 rows)`);
|
||||
this.name = 'LockStolenError';
|
||||
this.lockId = lockId;
|
||||
}
|
||||
}
|
||||
|
||||
/** Default TTL: 30 minutes, same as cycle lock. */
|
||||
@@ -205,7 +235,7 @@ export async function tryAcquireDbLock(
|
||||
// `gbrain sync --break-lock --max-age <s>` uses last_refreshed_at (not
|
||||
// acquired_at) to identify wedged-but-alive holders without stealing
|
||||
// healthy long-running holders that are actively refreshing.
|
||||
const rows: Array<{ id: string }> = await sql`
|
||||
const rows: Array<{ id: string; fence: string }> = await sql`
|
||||
INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
|
||||
VALUES (${lockId}, ${pid}, ${host}, NOW(), NOW() + ${ttl}::interval, NOW())
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
@@ -217,35 +247,47 @@ export async function tryAcquireDbLock(
|
||||
WHERE gbrain_cycle_locks.ttl_expires_at < NOW()
|
||||
AND (gbrain_cycle_locks.last_refreshed_at IS NULL
|
||||
OR gbrain_cycle_locks.last_refreshed_at < NOW() - ${stealGraceSeconds} * INTERVAL '1 second')
|
||||
RETURNING id
|
||||
RETURNING id, extract(epoch from acquired_at)::text AS fence
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
// Fencing identity (D5.10): acquired_at is written fresh on INSERT and on
|
||||
// every steal, so it uniquely names THIS acquisition. Rendered as
|
||||
// extract(epoch ...)::text — GUC-INDEPENDENT (ship-review catch, 3
|
||||
// specialists): plain timestamptz::text varies with per-session
|
||||
// TimeZone/DateStyle, and the fence is captured on the acquire pool but
|
||||
// compared on the direct pool; a GUC divergence would turn every fenced
|
||||
// refresh into a false steal. Epoch text is stable across sessions and
|
||||
// keeps microsecond precision.
|
||||
const fence = rows[0].fence;
|
||||
const deregister = registerCleanup(`db-lock:${lockId}`, async () => {
|
||||
await sql`
|
||||
DELETE FROM gbrain_cycle_locks
|
||||
WHERE id = ${lockId} AND holder_pid = ${pid}
|
||||
WHERE id = ${lockId} AND holder_pid = ${pid} AND extract(epoch from acquired_at)::text = ${fence}
|
||||
`;
|
||||
});
|
||||
return {
|
||||
id: lockId,
|
||||
acquiredAt: fence,
|
||||
refresh: async () => {
|
||||
// v0.41.13.0: bump BOTH ttl_expires_at AND last_refreshed_at.
|
||||
// v0.42.x (#1794): route through the DIRECT session pool, not the
|
||||
// transaction pool, so a Supavisor pooler exhaustion (EMAXCONNSESSION)
|
||||
// can't kill the heartbeat and let the live lock get stolen.
|
||||
await engine.executeRawDirect(
|
||||
const updated = await engine.executeRawDirect<{ id: string }>(
|
||||
`UPDATE gbrain_cycle_locks
|
||||
SET ttl_expires_at = NOW() + ($1)::interval,
|
||||
last_refreshed_at = NOW()
|
||||
WHERE id = $2 AND holder_pid = $3`,
|
||||
[ttl, lockId, pid],
|
||||
WHERE id = $2 AND holder_pid = $3 AND extract(epoch from acquired_at)::text = $4
|
||||
RETURNING id`,
|
||||
[ttl, lockId, pid, fence],
|
||||
);
|
||||
return updated.length > 0;
|
||||
},
|
||||
release: async () => {
|
||||
deregister();
|
||||
await sql`
|
||||
DELETE FROM gbrain_cycle_locks
|
||||
WHERE id = ${lockId} AND holder_pid = ${pid}
|
||||
WHERE id = ${lockId} AND holder_pid = ${pid} AND extract(epoch from acquired_at)::text = ${fence}
|
||||
`;
|
||||
},
|
||||
};
|
||||
@@ -266,32 +308,37 @@ export async function tryAcquireDbLock(
|
||||
WHERE gbrain_cycle_locks.ttl_expires_at < NOW()
|
||||
AND (gbrain_cycle_locks.last_refreshed_at IS NULL
|
||||
OR gbrain_cycle_locks.last_refreshed_at < NOW() - $5 * INTERVAL '1 second')
|
||||
RETURNING id`,
|
||||
RETURNING id, extract(epoch from acquired_at)::text AS fence`,
|
||||
[lockId, pid, host, ttl, stealGraceSeconds],
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
// Fencing identity (D5.10) — see the postgres branch for rationale.
|
||||
const fence = String((rows[0] as { fence: string }).fence);
|
||||
const deregister = registerCleanup(`db-lock:${lockId}`, async () => {
|
||||
await db.query(
|
||||
`DELETE FROM gbrain_cycle_locks WHERE id = $1 AND holder_pid = $2`,
|
||||
[lockId, pid],
|
||||
`DELETE FROM gbrain_cycle_locks WHERE id = $1 AND holder_pid = $2 AND extract(epoch from acquired_at)::text = $3`,
|
||||
[lockId, pid, fence],
|
||||
);
|
||||
});
|
||||
return {
|
||||
id: lockId,
|
||||
acquiredAt: fence,
|
||||
refresh: async () => {
|
||||
await db.query(
|
||||
const res = await db.query(
|
||||
`UPDATE gbrain_cycle_locks
|
||||
SET ttl_expires_at = NOW() + $1::interval,
|
||||
last_refreshed_at = NOW()
|
||||
WHERE id = $2 AND holder_pid = $3`,
|
||||
[ttl, lockId, pid],
|
||||
WHERE id = $2 AND holder_pid = $3 AND extract(epoch from acquired_at)::text = $4
|
||||
RETURNING id`,
|
||||
[ttl, lockId, pid, fence],
|
||||
);
|
||||
return res.rows.length > 0;
|
||||
},
|
||||
release: async () => {
|
||||
deregister();
|
||||
await db.query(
|
||||
`DELETE FROM gbrain_cycle_locks WHERE id = $1 AND holder_pid = $2`,
|
||||
[lockId, pid],
|
||||
`DELETE FROM gbrain_cycle_locks WHERE id = $1 AND holder_pid = $2 AND extract(epoch from acquired_at)::text = $3`,
|
||||
[lockId, pid, fence],
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -852,7 +899,20 @@ export async function withRefreshingLock<T>(
|
||||
const timeout = new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('refresh_timeout')), heartbeatTimeoutMs)
|
||||
);
|
||||
await Promise.race([handle.refresh(), timeout]);
|
||||
const stillOwned = await Promise.race([handle.refresh(), timeout]);
|
||||
if (stillOwned === false) {
|
||||
// W0 (D5.10): the fenced refresh matched 0 rows — the lock was
|
||||
// stolen or force-cleared. Further refreshes are pointless (and a
|
||||
// fenced refresh can never re-take the successor's row). Stop the
|
||||
// heartbeat and shout; the work itself keeps running (this wrapper
|
||||
// has no cancellation seam — W7 threads one through its consumers),
|
||||
// but mutual exclusion is GONE and the degraded-heartbeat exit
|
||||
// message names it.
|
||||
clearInterval(interval);
|
||||
healthOk = false;
|
||||
process.stderr.write(`[lock-refresh] ${lockId}: ${new LockStolenError(lockId).message} — heartbeat stopped, mutual exclusion lost\n`);
|
||||
return;
|
||||
}
|
||||
healthOk = true;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
|
||||
+34
-18
@@ -18,12 +18,44 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import type { Chunk, ChunkInput } from './types.ts';
|
||||
import { embedBatchWithBackoff, restampIfDemotedToTitleTier } from '../commands/embed.ts';
|
||||
import { wrapChunkTextsForStoredMode } from './embedding-context.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #3, CONFIRMED): the ONE carry-through field list for
|
||||
* re-embed upserts. upsertChunks writes these columns as EXCLUDED.<col>
|
||||
* (overwrite, not COALESCE), so any re-embed path that omits a field resets
|
||||
* it — omitting `modality` flipped every image chunk to modality='text',
|
||||
* silently zeroing the image search arm (filter `cc.modality = 'image'`).
|
||||
* Pre-fix this list existed twice: here (correct, with modality) and in
|
||||
* commands/embed.ts preserveCodeMetadata (missing modality — the bug). Both
|
||||
* consumers now share THIS list; embedding_image is deliberately NOT carried
|
||||
* (the upsert COALESCEs it, and getChunks returns the pgvector as a string
|
||||
* which upsertChunks would mis-serialize).
|
||||
*/
|
||||
export function carryChunkMetadata(
|
||||
loaded: Pick<Partial<Chunk>,
|
||||
'modality' | 'language' | 'symbol_name' | 'symbol_type' | 'start_line'
|
||||
| 'end_line' | 'parent_symbol_path' | 'doc_comment' | 'symbol_name_qualified'>,
|
||||
base: ChunkInput,
|
||||
): ChunkInput {
|
||||
return {
|
||||
...base,
|
||||
modality: loaded.modality ?? undefined,
|
||||
language: loaded.language ?? undefined,
|
||||
symbol_name: loaded.symbol_name ?? undefined,
|
||||
symbol_type: loaded.symbol_type ?? undefined,
|
||||
start_line: loaded.start_line ?? undefined,
|
||||
end_line: loaded.end_line ?? undefined,
|
||||
parent_symbol_path: loaded.parent_symbol_path ?? undefined,
|
||||
doc_comment: loaded.doc_comment ?? undefined,
|
||||
symbol_name_qualified: loaded.symbol_name_qualified ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Last visited (page_id, chunk_index) for keyset-resume across runs. */
|
||||
export interface StaleCursor {
|
||||
afterPageId: number;
|
||||
@@ -208,28 +240,12 @@ export async function embedStaleForSource(
|
||||
for (let j = 0; j < stale.length; j++) {
|
||||
staleIdxToEmbedding.set(stale[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
const merged: ChunkInput[] = existing.map((c) => ({
|
||||
const merged: ChunkInput[] = existing.map((c) => carryChunkMetadata(c, {
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: staleIdxToEmbedding.get(c.chunk_index) ?? undefined,
|
||||
token_count: c.token_count || Math.ceil(c.chunk_text.length / 4),
|
||||
// Carry through per-chunk metadata. upsertChunks writes these as
|
||||
// EXCLUDED.<col> (not COALESCE), so omitting them here resets image
|
||||
// rows to modality='text' (breaking the image search arm's
|
||||
// modality='image' filter) and wipes code-chunk symbol metadata on
|
||||
// every embed-stale pass. embedding_image is deliberately NOT
|
||||
// carried: the upsert COALESCEs it, and getChunks returns the
|
||||
// pgvector as a string which upsertChunks would mis-serialize.
|
||||
modality: c.modality ?? undefined,
|
||||
language: c.language ?? undefined,
|
||||
symbol_name: c.symbol_name ?? undefined,
|
||||
symbol_type: c.symbol_type ?? undefined,
|
||||
start_line: c.start_line ?? undefined,
|
||||
end_line: c.end_line ?? undefined,
|
||||
parent_symbol_path: c.parent_symbol_path ?? undefined,
|
||||
doc_comment: c.doc_comment ?? undefined,
|
||||
symbol_name_qualified: c.symbol_name_qualified ?? undefined,
|
||||
}));
|
||||
await observed(pacer, () => engine.upsertChunks(slug, merged, { sourceId: keySourceId }));
|
||||
// v0.41.31: stamp provenance only when EVERY chunk was stale (fully
|
||||
|
||||
+224
-137
@@ -11,7 +11,7 @@
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type {
|
||||
MinionJob, MinionJobInput, MinionJobStatus, InboxMessage, TokenUpdate,
|
||||
MinionQueueOpts, ChildDoneMessage, Attachment, AttachmentInput,
|
||||
MinionQueueOpts, ChildDoneMessage, ChildOutcome, Attachment, AttachmentInput,
|
||||
} from './types.ts';
|
||||
import { rowToMinionJob, rowToInboxMessage, rowToAttachment } from './types.ts';
|
||||
import { validateAttachment } from './attachments.ts';
|
||||
@@ -570,7 +570,7 @@ export class MinionQueue {
|
||||
// waiting-children whose last open child we just cancelled.
|
||||
for (const parentId of parentIds) {
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
@@ -807,15 +807,26 @@ export class MinionQueue {
|
||||
*/
|
||||
async handleTimeouts(): Promise<MinionJob[]> {
|
||||
return this.engine.transaction(async (tx) => {
|
||||
// #1737: count the timed-out run as a spent attempt (terminal, no retry).
|
||||
// Safe against double-count: the worker sweep runs handleStalled ->
|
||||
// handleTimeouts -> handleWallClockTimeouts sequentially and awaited, and
|
||||
// each guards on `status = 'active'`, so the first to set status='dead'
|
||||
// excludes the row from the later sweeps.
|
||||
//
|
||||
// W0 (D5.12): candidates are discovered with a plain read, PARENTS are
|
||||
// locked first in sorted order (matching failJob's parent-before-child
|
||||
// order), and the child UPDATE re-checks every predicate under a
|
||||
// SKIP LOCKED subselect — see killJobs() for the shared tail.
|
||||
const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null }>(
|
||||
`SELECT id, parent_job_id FROM minion_jobs
|
||||
WHERE status = 'active'
|
||||
AND timeout_at IS NOT NULL
|
||||
AND timeout_at < now()
|
||||
AND lock_until > now()`
|
||||
);
|
||||
if (candidates.length === 0) return [];
|
||||
await this.lockParentsOrdered(tx, candidates);
|
||||
const rows = await tx.executeRaw<Record<string, unknown>>(
|
||||
// #1737: count the timed-out run as a spent attempt (terminal, no retry),
|
||||
// mirroring handleWallClockTimeouts + handleStalled. handleTimeouts is the
|
||||
// FIRST killer to fire for the long-lane handlers (timeout_ms stamped at
|
||||
// submit), so without this the job reads `attempts: 0/N (started: N)`.
|
||||
// Safe against double-count: the worker sweep runs handleStalled ->
|
||||
// handleTimeouts -> handleWallClockTimeouts sequentially and awaited, and
|
||||
// each guards on `status = 'active'`, so the first to set status='dead'
|
||||
// excludes the row from the later sweeps.
|
||||
`UPDATE minion_jobs SET
|
||||
status = 'dead',
|
||||
error_text = 'timeout exceeded',
|
||||
@@ -824,60 +835,106 @@ export class MinionQueue {
|
||||
lock_until = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE status = 'active'
|
||||
AND timeout_at IS NOT NULL
|
||||
AND timeout_at < now()
|
||||
AND lock_until > now()
|
||||
RETURNING *`
|
||||
WHERE id IN (
|
||||
SELECT id FROM minion_jobs
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND status = 'active'
|
||||
AND timeout_at IS NOT NULL
|
||||
AND timeout_at < now()
|
||||
AND lock_until > now()
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING *`,
|
||||
[candidates.map(c => c.id)]
|
||||
);
|
||||
|
||||
// v0.15: emit child_done(outcome='timeout') for every timed-out job that
|
||||
// had a parent. Without this, an aggregator waiting for N child_done
|
||||
// messages hangs forever when a child times out (codex iteration 3).
|
||||
// Outcome 'timeout' is distinct from 'dead' so consumers can distinguish
|
||||
// "timed out during run" from "died via max-stall".
|
||||
const parentIds = new Set<number>();
|
||||
for (const r of rows) {
|
||||
const parentJobId = r.parent_job_id as number | null;
|
||||
if (parentJobId == null) continue;
|
||||
parentIds.add(parentJobId);
|
||||
const childDone: ChildDoneMessage = {
|
||||
type: 'child_done',
|
||||
child_id: r.id as number,
|
||||
job_name: r.name as string,
|
||||
result: null,
|
||||
outcome: 'timeout',
|
||||
error: 'timeout exceeded',
|
||||
};
|
||||
await tx.executeRaw(
|
||||
`INSERT INTO minion_inbox (job_id, sender, payload)
|
||||
SELECT $1, 'minions', $2::jsonb
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
|
||||
)`,
|
||||
[parentJobId, childDone]
|
||||
);
|
||||
}
|
||||
|
||||
// Unblock any aggregator parents whose last open child we just killed.
|
||||
for (const parentId of parentIds) {
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE parent_job_id = $1
|
||||
AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
|
||||
)`,
|
||||
[parentId]
|
||||
);
|
||||
}
|
||||
|
||||
await this.killJobs(tx, rows, 'timeout', 'timeout exceeded');
|
||||
return rows.map(rowToMinionJob);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #4, D5.12): the ONE parent-notification tail shared
|
||||
* by every reaper that terminally kills active jobs. Pre-fix this ~45-line
|
||||
* block was hand-copied between handleTimeouts and handleWallClockTimeouts
|
||||
* (differing only in the error string), and handleStalled's dead-letter
|
||||
* branch had NO copy at all — a child that died via max-stall left its
|
||||
* aggregator parent in 'waiting-children' forever (the exact hang the v0.15
|
||||
* comment says was fixed for timeouts).
|
||||
*
|
||||
* Runs inside the caller's transaction, AFTER the child transitions.
|
||||
* Callers must have locked the parents first via lockParentsOrdered() —
|
||||
* parents-before-children is the queue-wide lock order (failJob locks the
|
||||
* parent before touching the child), so the reapers can never deadlock
|
||||
* against a concurrent failJob/completeJob.
|
||||
*
|
||||
* Emits child_done(outcome) to each non-terminal parent's inbox, then flips
|
||||
* any 'waiting-children' parent whose last open child we just killed back
|
||||
* to 'waiting'.
|
||||
*/
|
||||
private async killJobs(
|
||||
tx: Pick<BrainEngine, 'executeRaw'>,
|
||||
rows: Array<Record<string, unknown>>,
|
||||
outcome: ChildOutcome,
|
||||
errorText: string,
|
||||
): Promise<void> {
|
||||
const parentIds = new Set<number>();
|
||||
for (const r of rows) {
|
||||
const parentJobId = r.parent_job_id as number | null;
|
||||
if (parentJobId == null) continue;
|
||||
parentIds.add(parentJobId);
|
||||
const childDone: ChildDoneMessage = {
|
||||
type: 'child_done',
|
||||
child_id: r.id as number,
|
||||
job_name: r.name as string,
|
||||
result: null,
|
||||
outcome,
|
||||
error: errorText,
|
||||
};
|
||||
await tx.executeRaw(
|
||||
`INSERT INTO minion_inbox (job_id, sender, payload)
|
||||
SELECT $1, 'minions', $2::jsonb
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
|
||||
)`,
|
||||
[parentJobId, childDone]
|
||||
);
|
||||
}
|
||||
|
||||
// Unblock any aggregator parents whose last open child we just killed.
|
||||
for (const parentId of [...parentIds].sort((a, b) => a - b)) {
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE parent_job_id = $1
|
||||
AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
|
||||
)`,
|
||||
[parentId]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* W0 (D5.12): take parent row locks in ASCENDING id order before any child
|
||||
* transition. Matches failJob's parent-first order so the three reapers and
|
||||
* failJob can never deadlock each other on parent/child lock acquisition.
|
||||
*/
|
||||
private async lockParentsOrdered(
|
||||
tx: Pick<BrainEngine, 'executeRaw'>,
|
||||
candidates: Array<{ parent_job_id: number | null }>,
|
||||
): Promise<void> {
|
||||
const parentIds = [...new Set(
|
||||
candidates.map(c => c.parent_job_id).filter((p): p is number => p != null),
|
||||
)].sort((a, b) => a - b);
|
||||
if (parentIds.length === 0) return;
|
||||
await tx.executeRaw(
|
||||
`SELECT id FROM minion_jobs WHERE id = ANY($1::bigint[]) ORDER BY id FOR UPDATE`,
|
||||
[parentIds]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dead-letter active jobs that exceed a wall-clock runtime threshold,
|
||||
* regardless of lock state. This catches jobs stuck while still holding
|
||||
@@ -889,6 +946,21 @@ export class MinionQueue {
|
||||
*/
|
||||
async handleWallClockTimeouts(lockDurationMs: number): Promise<MinionJob[]> {
|
||||
return this.engine.transaction(async (tx) => {
|
||||
// W0 (D5.12): same parents-first discover/lock/kill shape as
|
||||
// handleTimeouts; shared tail in killJobs().
|
||||
const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null }>(
|
||||
`SELECT id, parent_job_id FROM minion_jobs
|
||||
WHERE status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND EXTRACT(EPOCH FROM (now() - started_at)) * 1000 >
|
||||
CASE
|
||||
WHEN timeout_ms IS NOT NULL THEN timeout_ms * 2
|
||||
ELSE $1::double precision * 2 * GREATEST(max_stalled, 1)
|
||||
END`,
|
||||
[lockDurationMs]
|
||||
);
|
||||
if (candidates.length === 0) return [];
|
||||
await this.lockParentsOrdered(tx, candidates);
|
||||
const rows = await tx.executeRaw<Record<string, unknown>>(
|
||||
`UPDATE minion_jobs SET
|
||||
status = 'dead',
|
||||
@@ -898,54 +970,22 @@ export class MinionQueue {
|
||||
lock_until = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND EXTRACT(EPOCH FROM (now() - started_at)) * 1000 >
|
||||
CASE
|
||||
WHEN timeout_ms IS NOT NULL THEN timeout_ms * 2
|
||||
ELSE $1::double precision * 2 * GREATEST(max_stalled, 1)
|
||||
END
|
||||
WHERE id IN (
|
||||
SELECT id FROM minion_jobs
|
||||
WHERE id = ANY($2::bigint[])
|
||||
AND status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND EXTRACT(EPOCH FROM (now() - started_at)) * 1000 >
|
||||
CASE
|
||||
WHEN timeout_ms IS NOT NULL THEN timeout_ms * 2
|
||||
ELSE $1::double precision * 2 * GREATEST(max_stalled, 1)
|
||||
END
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING *`,
|
||||
[lockDurationMs]
|
||||
[lockDurationMs, candidates.map(c => c.id)]
|
||||
);
|
||||
|
||||
const parentIds = new Set<number>();
|
||||
for (const r of rows) {
|
||||
const parentJobId = r.parent_job_id as number | null;
|
||||
if (parentJobId == null) continue;
|
||||
parentIds.add(parentJobId);
|
||||
const childDone: ChildDoneMessage = {
|
||||
type: 'child_done',
|
||||
child_id: r.id as number,
|
||||
job_name: r.name as string,
|
||||
result: null,
|
||||
outcome: 'timeout',
|
||||
error: 'wall-clock timeout exceeded',
|
||||
};
|
||||
await tx.executeRaw(
|
||||
`INSERT INTO minion_inbox (job_id, sender, payload)
|
||||
SELECT $1, 'minions', $2::jsonb
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
|
||||
)`,
|
||||
[parentJobId, childDone]
|
||||
);
|
||||
}
|
||||
|
||||
for (const parentId of parentIds) {
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
WHERE parent_job_id = $1
|
||||
AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
|
||||
)`,
|
||||
[parentId]
|
||||
);
|
||||
}
|
||||
|
||||
await this.killJobs(tx, rows, 'timeout', 'wall-clock timeout exceeded');
|
||||
return rows.map(rowToMinionJob);
|
||||
});
|
||||
}
|
||||
@@ -1034,7 +1074,7 @@ export class MinionQueue {
|
||||
// child with on_child_fail='continue'/'ignore' doesn't strand the
|
||||
// parent in waiting-children forever (v0.15 aggregator fix).
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
@@ -1101,6 +1141,7 @@ export class MinionQueue {
|
||||
stacktrace = COALESCE(stacktrace, '[]'::jsonb) || to_jsonb($3::text),
|
||||
delay_until = CASE WHEN $1 = 'delayed' THEN now() + ($4::double precision * interval '1 millisecond') ELSE NULL END,
|
||||
finished_at = CASE WHEN $1 IN ('failed', 'dead') THEN now() ELSE NULL END,
|
||||
started_at = CASE WHEN $1 = 'delayed' THEN NULL ELSE started_at END,
|
||||
lock_token = NULL, lock_until = NULL, updated_at = now()
|
||||
WHERE id = $5 AND status = 'active' AND lock_token = $6
|
||||
RETURNING *`,
|
||||
@@ -1151,7 +1192,7 @@ export class MinionQueue {
|
||||
// After dropping the dep, try to resolve the parent if all OTHER
|
||||
// kids are terminal. Terminal set includes 'failed' (v0.15).
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
@@ -1169,7 +1210,7 @@ export class MinionQueue {
|
||||
// remain. Run the resolve check here so the last child transitioning
|
||||
// via THIS code path still unblocks the parent.
|
||||
await tx.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
@@ -1229,6 +1270,7 @@ export class MinionQueue {
|
||||
error_text = $1,
|
||||
stacktrace = COALESCE(stacktrace, '[]'::jsonb) || to_jsonb($1::text),
|
||||
delay_until = now() + ($2::double precision * interval '1 millisecond'),
|
||||
started_at = NULL,
|
||||
lock_token = NULL, lock_until = NULL, updated_at = now()
|
||||
WHERE id = $3 AND status = 'active' AND lock_token = $4
|
||||
RETURNING *`,
|
||||
@@ -1304,6 +1346,7 @@ export class MinionQueue {
|
||||
async promoteDelayed(): Promise<MinionJob[]> {
|
||||
const rows = await this.lockRetry(() => this.engine.executeRaw<Record<string, unknown>>(
|
||||
`UPDATE minion_jobs SET status = 'waiting', delay_until = NULL,
|
||||
started_at = NULL,
|
||||
lock_token = NULL, lock_until = NULL, updated_at = now()
|
||||
WHERE status = 'delayed' AND delay_until <= now()
|
||||
RETURNING *`
|
||||
@@ -1313,40 +1356,84 @@ export class MinionQueue {
|
||||
|
||||
/** Detect and handle stalled jobs. Single CTE, no off-by-one. Returns affected jobs. */
|
||||
async handleStalled(): Promise<{ requeued: MinionJob[]; dead: MinionJob[] }> {
|
||||
const rows = await this.engine.executeRaw<Record<string, unknown> & { action: string }>(
|
||||
`WITH stalled AS (
|
||||
SELECT id, stalled_counter, max_stalled
|
||||
FROM minion_jobs
|
||||
WHERE status = 'active' AND lock_until < now()
|
||||
FOR UPDATE SKIP LOCKED
|
||||
),
|
||||
requeued AS (
|
||||
UPDATE minion_jobs SET
|
||||
// W0 fix-wave (Tier-1 #4): the dead-letter branch previously emitted NO
|
||||
// child_done and never unblocked aggregator parents — a child that died
|
||||
// via max-stall stranded its parent in 'waiting-children' forever (the
|
||||
// exact hang the v0.15 comment says was fixed for timeouts; there was no
|
||||
// compensating sweep anywhere). Restructured into the parents-first
|
||||
// discover/lock/kill shape (D5.12) with the shared killJobs() tail.
|
||||
return this.engine.transaction(async (tx) => {
|
||||
const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null; stalled_counter: number; max_stalled: number }>(
|
||||
`SELECT id, parent_job_id, stalled_counter, max_stalled
|
||||
FROM minion_jobs
|
||||
WHERE status = 'active' AND lock_until < now()`
|
||||
);
|
||||
if (candidates.length === 0) return { requeued: [], dead: [] };
|
||||
const ids = candidates.map(c => c.id);
|
||||
// Only the dead-letter branch touches parents; lock just those, sorted.
|
||||
await this.lockParentsOrdered(
|
||||
tx,
|
||||
candidates.filter(c => Number(c.stalled_counter) + 1 >= Number(c.max_stalled)),
|
||||
);
|
||||
|
||||
const requeuedRows = await tx.executeRaw<Record<string, unknown>>(
|
||||
`UPDATE minion_jobs SET
|
||||
status = 'waiting', stalled_counter = stalled_counter + 1,
|
||||
started_at = NULL,
|
||||
lock_token = NULL, lock_until = NULL, updated_at = now()
|
||||
WHERE id IN (SELECT id FROM stalled WHERE stalled_counter + 1 < max_stalled)
|
||||
RETURNING *, 'requeued' as action
|
||||
),
|
||||
dead_lettered AS (
|
||||
UPDATE minion_jobs SET
|
||||
WHERE id IN (
|
||||
SELECT id FROM minion_jobs
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND status = 'active' AND lock_until < now()
|
||||
AND stalled_counter + 1 < max_stalled
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING *`,
|
||||
[ids]
|
||||
);
|
||||
const deadRows = await tx.executeRaw<Record<string, unknown>>(
|
||||
`UPDATE minion_jobs SET
|
||||
status = 'dead', stalled_counter = stalled_counter + 1,
|
||||
attempts_made = attempts_made + 1,
|
||||
error_text = 'max stalled count exceeded',
|
||||
lock_token = NULL, lock_until = NULL, finished_at = now(), updated_at = now()
|
||||
WHERE id IN (SELECT id FROM stalled WHERE stalled_counter + 1 >= max_stalled)
|
||||
RETURNING *, 'dead' as action
|
||||
)
|
||||
SELECT * FROM requeued UNION ALL SELECT * FROM dead_lettered`
|
||||
);
|
||||
|
||||
const requeued: MinionJob[] = [];
|
||||
const dead: MinionJob[] = [];
|
||||
for (const r of rows) {
|
||||
const job = rowToMinionJob(r);
|
||||
if (r.action === 'requeued') requeued.push(job);
|
||||
else dead.push(job);
|
||||
}
|
||||
return { requeued, dead };
|
||||
WHERE id IN (
|
||||
SELECT id FROM minion_jobs
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND status = 'active' AND lock_until < now()
|
||||
AND stalled_counter + 1 >= max_stalled
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING *`,
|
||||
[ids]
|
||||
);
|
||||
// THE FIX: stall-death now notifies + unblocks parents like every
|
||||
// other terminal kill. Outcome 'dead' (not 'timeout') so consumers can
|
||||
// distinguish "died via max-stall" from "timed out during run".
|
||||
await this.killJobs(tx, deadRows, 'dead', 'max stalled count exceeded');
|
||||
return { requeued: requeuedRows.map(rowToMinionJob), dead: deadRows.map(rowToMinionJob) };
|
||||
}).then(async (result) => {
|
||||
// W0 ship-review (data-migration): the per-kill unblock above is
|
||||
// forward-only — parents stranded in 'waiting-children' by PRE-upgrade
|
||||
// stall-deaths (children already status='dead') are never revisited by
|
||||
// any per-event unblock site. This idempotent sweep self-heals ALL
|
||||
// stranding classes, retroactive included, once per stall tick: any
|
||||
// waiting-children parent with zero non-terminal children flips back to
|
||||
// 'waiting'. Cheap (single UPDATE, NOT EXISTS on an indexed FK) at the
|
||||
// 30s sweep cadence.
|
||||
try {
|
||||
await this.engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs c
|
||||
WHERE c.parent_job_id = minion_jobs.id
|
||||
AND c.status NOT IN ('completed', 'failed', 'dead', 'cancelled')
|
||||
)`
|
||||
);
|
||||
} catch { /* best-effort backstop; the per-kill unblock is the primary path */ }
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1358,7 +1445,7 @@ export class MinionQueue {
|
||||
*/
|
||||
async resolveParent(parentId: number): Promise<MinionJob | null> {
|
||||
const rows = await this.engine.executeRaw<Record<string, unknown>>(
|
||||
`UPDATE minion_jobs SET status = 'waiting', updated_at = now()
|
||||
`UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'waiting-children'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minion_jobs
|
||||
|
||||
@@ -839,7 +839,22 @@ export class MinionSupervisor {
|
||||
private async refreshDbLock(): Promise<void> {
|
||||
if (!this.dbLock || this.stopping) return;
|
||||
try {
|
||||
await this.dbLock.refresh();
|
||||
const stillOwned = await this.dbLock.refresh();
|
||||
if (stillOwned === false) {
|
||||
// W0 fix-wave (D5.10): the fenced refresh matched 0 rows — the lock
|
||||
// was stolen or force-cleared. That is CERTAIN loss, not a blip:
|
||||
// counting it toward the failure threshold (or worse, resetting the
|
||||
// counter as a "success") would let two supervisors drain the same
|
||||
// queue for up to two more refresh windows. Exit immediately; the
|
||||
// process manager restarts a single clean supervisor.
|
||||
this.emit('health_error', {
|
||||
reason: 'supervisor_lock_lost',
|
||||
detail: 'fenced refresh matched 0 rows (stolen or force-cleared)',
|
||||
queue: this.opts.queue,
|
||||
});
|
||||
await this.shutdown('supervisor_lock_lost', ExitCodes.LOCK_LOST);
|
||||
return;
|
||||
}
|
||||
this.lockRefreshFailures = 0;
|
||||
} catch (e) {
|
||||
this.lockRefreshFailures++;
|
||||
|
||||
@@ -117,14 +117,14 @@ const PGLITE_EDGE_BATCH_MAX_BIND_PARAMS = 30_000;
|
||||
// silently fall through to a normal initSchema (snapshot is just an
|
||||
// optimization, never authoritative).
|
||||
let _snapshotWarnLogged = false;
|
||||
function tryLoadSnapshot(snapshotPath: string): Blob | null {
|
||||
export function tryLoadSnapshot(snapshotPath: string): Blob | null {
|
||||
try {
|
||||
// Lazy require so production builds without these imports don't crash.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const fs = require('node:fs') as typeof import('node:fs');
|
||||
const crypto = require('node:crypto') as typeof import('node:crypto');
|
||||
const { MIGRATIONS } = require('./migrate.ts') as typeof import('./migrate.ts');
|
||||
const { PGLITE_SCHEMA_SQL } = require('./pglite-schema.ts') as typeof import('./pglite-schema.ts');
|
||||
const fs = require('node:fs') as typeof import('node:fs'); // engine-dynamic-import-ok
|
||||
const crypto = require('node:crypto') as typeof import('node:crypto'); // engine-dynamic-import-ok
|
||||
const { MIGRATIONS } = require('./migrate.ts') as typeof import('./migrate.ts'); // engine-dynamic-import-ok
|
||||
const { PGLITE_SCHEMA_SQL } = require('./pglite-schema.ts') as typeof import('./pglite-schema.ts'); // engine-dynamic-import-ok
|
||||
|
||||
if (!fs.existsSync(snapshotPath)) {
|
||||
if (!_snapshotWarnLogged) {
|
||||
@@ -144,7 +144,32 @@ function tryLoadSnapshot(snapshotPath: string): Blob | null {
|
||||
return null;
|
||||
}
|
||||
const expectedHash = computeSnapshotSchemaHash(MIGRATIONS, PGLITE_SCHEMA_SQL, crypto);
|
||||
const actualHash = fs.readFileSync(versionPath, 'utf8').trim();
|
||||
const versionLines = fs.readFileSync(versionPath, 'utf8').trim().split('\n');
|
||||
const actualHash = versionLines[0] ?? '';
|
||||
|
||||
// W0 fix-wave: the version file's dims=/model= lines record the embedding
|
||||
// shape the snapshot was BAKED with. A snapshot whose vector(dims) columns
|
||||
// differ from what THIS process would create poisons every embedding
|
||||
// write ("expected 1280 dimensions, not 1536" — the W0 incident when the
|
||||
// fixture went default-on). Resolve our would-be shape through the same
|
||||
// gateway-with-default fallback initSchema uses and refuse a mismatch.
|
||||
// Version files without the shape lines (pre-W0) are treated as stale.
|
||||
let wantDims: number | string = DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
let wantModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
const gw = require('./ai/gateway.ts') as typeof import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
wantDims = gw.getEmbeddingDimensions();
|
||||
wantModel = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — defaults, same as initSchema */ }
|
||||
const shapeOk = versionLines[1] === `dims=${wantDims}` && versionLines[2] === `model=${wantModel}`;
|
||||
if (!shapeOk) {
|
||||
if (!_snapshotWarnLogged) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`[pglite] snapshot embedding shape mismatch (want dims=${wantDims} model=${wantModel}, have ${versionLines[1] ?? 'none'} ${versionLines[2] ?? ''}) — using normal init. Rebuild with: bun run build:pglite-snapshot`);
|
||||
_snapshotWarnLogged = true;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (expectedHash !== actualHash) {
|
||||
if (!_snapshotWarnLogged) {
|
||||
// eslint-disable-next-line no-console
|
||||
@@ -162,7 +187,7 @@ function tryLoadSnapshot(snapshotPath: string): Blob | null {
|
||||
}
|
||||
|
||||
export function computeSnapshotSchemaHash(
|
||||
migrations: Array<{ version: number; name: string; sql?: string; sqlFor?: { pglite?: string } }>,
|
||||
migrations: Array<{ version: number; name: string; sql?: string; sqlFor?: { pglite?: string }; handler?: unknown }>,
|
||||
schemaSQL: string,
|
||||
crypto: typeof import('node:crypto'),
|
||||
): string {
|
||||
@@ -178,6 +203,13 @@ export function computeSnapshotSchemaHash(
|
||||
hash.update(m.sql ?? '');
|
||||
hash.update('\t');
|
||||
hash.update(m.sqlFor?.pglite ?? '');
|
||||
hash.update('\t');
|
||||
// W0 fix-wave (D5.13, Codex #4): 19+ migrations carry executable
|
||||
// `handler` code with empty/absent sql — invisible to the sql-only hash,
|
||||
// so editing a handler reused a stale snapshot. Function.prototype
|
||||
// .toString folds the handler SOURCE into the hash (deterministic within
|
||||
// a checkout; this is a dev/test fixture, not a shipped artifact).
|
||||
hash.update(typeof m.handler === 'function' ? String(m.handler) : '');
|
||||
hash.update('\n');
|
||||
}
|
||||
return hash.digest('hex');
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# gbrain agent workspace — template
|
||||
|
||||
<!-- gbrain-template-stamp: 0.45.15.0 -->
|
||||
<!-- gbrain-template-stamp: 0.45.16.0 -->
|
||||
|
||||
This repository is the **"Use this template"** distribution artifact for a
|
||||
[gbrain](https://github.com/garrytan/gbrain) personal-agent workspace — the same
|
||||
|
||||
@@ -142,14 +142,19 @@ describe('autopilot-cycle handler contract (v0.20.5)', () => {
|
||||
'utf8',
|
||||
);
|
||||
|
||||
// Count checkAborted calls in the runCycle function body
|
||||
// Count checkAborted calls in the runCycle function body. W0 fix-wave:
|
||||
// boundaries check `cycleSignal` — the combined external-caller +
|
||||
// lock-steal signal — so a stolen lock aborts at the same seams an
|
||||
// external abort always did.
|
||||
const runCycleBody = cycleSource.slice(
|
||||
cycleSource.indexOf('export async function runCycle'),
|
||||
);
|
||||
const checkCalls = (runCycleBody.match(/checkAborted\(opts\.signal\)/g) || []).length;
|
||||
const checkCalls = (runCycleBody.match(/checkAborted\(cycleSignal\)/g) || []).length;
|
||||
|
||||
// Should have at least 6 (one per phase)
|
||||
expect(checkCalls).toBeGreaterThanOrEqual(6);
|
||||
// And the combined signal must actually fold BOTH sources.
|
||||
expect(runCycleBody).toContain('anyAbortSignal([externalSignal, stolen.signal])');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,11 +179,12 @@ describe('#1972 — complete cooperative-abort coverage', () => {
|
||||
const fs = await import('fs');
|
||||
const src = fs.readFileSync(new URL('../src/core/cycle.ts', import.meta.url), 'utf8');
|
||||
const body = src.slice(src.indexOf('export async function runCycle'));
|
||||
// Each long phase receives the signal.
|
||||
expect(body).toContain('runPhaseExtract(engine, brainDir, dryRun, syncPagesAffected, opts.signal, cycleSourceId)');
|
||||
expect(body).toMatch(/runPhaseExtractFacts\([^)]*opts\.signal\)/);
|
||||
expect(body).toContain('signal: opts.signal'); // consolidate opts
|
||||
expect(body).toContain('runPhaseLint(brainDir, dryRun, engine, opts.signal)');
|
||||
// Each long phase receives the signal (W0: the combined cycleSignal, so
|
||||
// phases also stop on lock-steal, not just external aborts).
|
||||
expect(body).toContain('runPhaseExtract(engine, brainDir, dryRun, syncPagesAffected, cycleSignal, cycleSourceId)');
|
||||
expect(body).toMatch(/runPhaseExtractFacts\([^)]*cycleSignal\)/);
|
||||
expect(body).toContain('signal: cycleSignal'); // consolidate opts
|
||||
expect(body).toContain('runPhaseLint(brainDir, dryRun, engine, cycleSignal)');
|
||||
// Reaper runs at cycle start.
|
||||
expect(body).toContain('reapDeadHolderLocks(engine)');
|
||||
// Terminal guard: the success stamp is gated on !aborted, and the report
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* W0 ship-review coverage — anyAbortSignal behavioral tests (the combined
|
||||
* external-caller + lock-steal signal). Previously pinned only by a
|
||||
* source-string contract assertion.
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { anyAbortSignal } from '../src/core/cycle.ts';
|
||||
|
||||
test('pre-aborted input aborts the combined signal immediately with the same reason', () => {
|
||||
const c = new AbortController();
|
||||
const why = new Error('already done');
|
||||
c.abort(why);
|
||||
const combined = anyAbortSignal([c.signal, new AbortController().signal]);
|
||||
try {
|
||||
expect(combined.signal.aborted).toBe(true);
|
||||
expect(combined.signal.reason).toBe(why);
|
||||
} finally {
|
||||
combined.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test('a real signal aborting later propagates reason to the combined signal', async () => {
|
||||
const a = new AbortController();
|
||||
const b = new AbortController();
|
||||
const combined = anyAbortSignal([a.signal, b.signal]);
|
||||
try {
|
||||
expect(combined.signal.aborted).toBe(false);
|
||||
const why = new Error('steal');
|
||||
b.abort(why);
|
||||
expect(combined.signal.aborted).toBe(true);
|
||||
expect(combined.signal.reason).toBe(why);
|
||||
} finally {
|
||||
combined.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test('duck-typed stub (no addEventListener) is observed via the 50ms poll', async () => {
|
||||
const stub = { aborted: false } as unknown as AbortSignal;
|
||||
const real = new AbortController();
|
||||
const combined = anyAbortSignal([stub, real.signal]);
|
||||
try {
|
||||
expect(combined.signal.aborted).toBe(false);
|
||||
(stub as { aborted: boolean }).aborted = true;
|
||||
const deadline = Date.now() + 2_000;
|
||||
while (!combined.signal.aborted && Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 25));
|
||||
}
|
||||
expect(combined.signal.aborted).toBe(true);
|
||||
} finally {
|
||||
combined.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test('dispose() detaches the forward listener from the CALLER signal (the daemon leak class)', () => {
|
||||
const daemonSignal = new AbortController();
|
||||
// Simulate many cycle ticks against one daemon-lifetime signal.
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const combined = anyAbortSignal([daemonSignal.signal, new AbortController().signal]);
|
||||
combined.dispose();
|
||||
}
|
||||
// After dispose, aborting the daemon signal must not touch the (disposed)
|
||||
// combined controllers — and no listener buildup means no
|
||||
// MaxListenersExceededWarning. Behavioral proxy: a fresh combine still
|
||||
// works and the abort propagates exactly once.
|
||||
const last = anyAbortSignal([daemonSignal.signal]);
|
||||
try {
|
||||
daemonSignal.abort(new Error('shutdown'));
|
||||
expect(last.signal.aborted).toBe(true);
|
||||
} finally {
|
||||
last.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test('dispose() clears the stub poll interval (no immortal timers)', async () => {
|
||||
const stub = { aborted: false } as unknown as AbortSignal;
|
||||
const combined = anyAbortSignal([stub]);
|
||||
combined.dispose();
|
||||
// Flip after dispose: the poll must be dead — combined never aborts.
|
||||
(stub as { aborted: boolean }).aborted = true;
|
||||
await new Promise(r => setTimeout(r, 150));
|
||||
expect(combined.signal.aborted).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #1, D5.6 abort-determinism) — end-to-end lock-steal
|
||||
* abort through runCycle.
|
||||
*
|
||||
* A cycle whose lock row is stolen mid-run must:
|
||||
* 1. detect the steal via the background refresher (fenced refresh → false),
|
||||
* 2. stop at the next phase boundary (no further phases run),
|
||||
* 3. return a STRUCTURED partial report (reason 'lock_stolen') instead of
|
||||
* throwing — completed phases' results are present (their DB writes are
|
||||
* durable, persisted-and-resumable),
|
||||
* 4. leave the successor's lock row intact (release is fenced).
|
||||
*
|
||||
* Serial: mutates GBRAIN_HOME + GBRAIN_CYCLE_LOCK_REFRESH_MS.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runCycle } from '../src/core/cycle.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let gbrainHome: string;
|
||||
let brainDir: string;
|
||||
const PRIOR_HOME = process.env.GBRAIN_HOME;
|
||||
const PRIOR_REFRESH = process.env.GBRAIN_CYCLE_LOCK_REFRESH_MS;
|
||||
|
||||
beforeAll(async () => {
|
||||
// GBRAIN_HOME isolation so the PGLite file lock doesn't touch the real
|
||||
// ~/.gbrain/cycle.lock (same pattern as cycle-pglite-lock-ordering).
|
||||
gbrainHome = mkdtempSync(join(tmpdir(), 'gbrain-steal-home-'));
|
||||
brainDir = mkdtempSync(join(tmpdir(), 'gbrain-steal-brain-'));
|
||||
process.env.GBRAIN_HOME = gbrainHome;
|
||||
// Deterministic fast refresher: 20ms tick against the 200ms in-test sleep.
|
||||
process.env.GBRAIN_CYCLE_LOCK_REFRESH_MS = '20';
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
if (PRIOR_HOME === undefined) delete process.env.GBRAIN_HOME; else process.env.GBRAIN_HOME = PRIOR_HOME;
|
||||
if (PRIOR_REFRESH === undefined) delete process.env.GBRAIN_CYCLE_LOCK_REFRESH_MS; else process.env.GBRAIN_CYCLE_LOCK_REFRESH_MS = PRIOR_REFRESH;
|
||||
rmSync(gbrainHome, { recursive: true, force: true });
|
||||
rmSync(brainDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('mid-run steal → structured partial report, no further phases, successor row intact', async () => {
|
||||
const lockId = 'gbrain-cycle:stealtest';
|
||||
let stole = false;
|
||||
|
||||
const report = await runCycle(engine, {
|
||||
brainDir,
|
||||
sourceId: 'stealtest',
|
||||
phases: ['lint', 'backlinks'],
|
||||
yieldBetweenPhases: async () => {
|
||||
if (stole) return;
|
||||
stole = true;
|
||||
// Simulate a successor taking the row: new acquisition identity.
|
||||
await engine.executeRaw(
|
||||
`UPDATE gbrain_cycle_locks
|
||||
SET acquired_at = acquired_at + INTERVAL '1 millisecond',
|
||||
ttl_expires_at = NOW() + INTERVAL '5 minutes',
|
||||
last_refreshed_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[lockId],
|
||||
);
|
||||
// Give the 20ms refresher a 1.5s window (75 nominal ticks) to observe
|
||||
// the steal — sized for shard-load timer starvation, the flake class
|
||||
// db-lock-fencing.test.ts documents (a 200ms window ≈ 10 ticks proved
|
||||
// too tight under a loaded suite).
|
||||
await new Promise(r => setTimeout(r, 1_500));
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.status).toBe('partial');
|
||||
expect(report.reason).toBe('lock_stolen');
|
||||
// lint completed before the steal; backlinks must never have started.
|
||||
expect(report.phases.map(p => p.phase)).toEqual(['lint']);
|
||||
|
||||
// The fenced release must NOT have deleted the successor's row.
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM gbrain_cycle_locks WHERE id = $1`,
|
||||
[lockId],
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
// Cleanup for other suites.
|
||||
await engine.executeRaw(`DELETE FROM gbrain_cycle_locks WHERE id = $1`, [lockId]);
|
||||
});
|
||||
|
||||
test('steal-free cycle still completes and releases normally (regression guard)', async () => {
|
||||
const report = await runCycle(engine, {
|
||||
brainDir,
|
||||
sourceId: 'stealtest2',
|
||||
phases: ['lint', 'backlinks'],
|
||||
});
|
||||
expect(report.status === 'ok' || report.status === 'clean' || report.status === 'partial').toBe(true);
|
||||
expect(report.reason).not.toBe('lock_stolen');
|
||||
expect(report.phases.map(p => p.phase)).toEqual(['lint', 'backlinks']);
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM gbrain_cycle_locks WHERE id = $1`,
|
||||
['gbrain-cycle:stealtest2'],
|
||||
);
|
||||
expect(rows.length).toBe(0);
|
||||
});
|
||||
@@ -15,7 +15,7 @@ import type { LockHandle } from '../../src/core/cycle.ts';
|
||||
function makeMockLock(): { lock: LockHandle; refreshCount: number; releaseCount: number } {
|
||||
const state = { refreshCount: 0, releaseCount: 0 };
|
||||
const lock: LockHandle = {
|
||||
refresh: async () => { state.refreshCount++; },
|
||||
refresh: async () => { state.refreshCount++; return true; },
|
||||
release: async () => { state.releaseCount++; },
|
||||
};
|
||||
return {
|
||||
@@ -60,7 +60,7 @@ describe('buildYieldDuringPhase (T3 codex fix)', () => {
|
||||
const outer = async () => { callOrder.push('outer'); };
|
||||
const fn = buildYieldDuringPhase({
|
||||
...tracker.lock,
|
||||
refresh: async () => { callOrder.push('refresh'); tracker.lock.refresh(); },
|
||||
refresh: async () => { callOrder.push('refresh'); return tracker.lock.refresh(); },
|
||||
}, outer);
|
||||
await fn!();
|
||||
expect(callOrder).toEqual(['refresh', 'outer']);
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #1, D5.10) — per-acquisition lock fencing.
|
||||
*
|
||||
* The refresh/release predicates require (id, holder_pid, acquired_at::text),
|
||||
* so a handle from a PREVIOUS acquisition — a PID-reuse impostor, or this
|
||||
* process after its row was stolen — can never refresh or delete a
|
||||
* successor's row. Pre-fix the predicates were (id, holder_pid) only, and an
|
||||
* expired holder could refresh a successor's lock while reporting success
|
||||
* (Codex eng-review #1 on the fix-wave plan).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { tryAcquireDbLock, LockStolenError } from '../src/core/db-lock.ts';
|
||||
import { startCycleLockRefresher, buildYieldDuringPhase, type LockHandle } from '../src/core/cycle.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM gbrain_cycle_locks WHERE id LIKE 'test-fence-%'`);
|
||||
});
|
||||
|
||||
/** Simulate a successor stealing the row: rewrite acquisition identity in place. */
|
||||
async function stealRow(lockId: string): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`UPDATE gbrain_cycle_locks
|
||||
SET holder_pid = holder_pid,
|
||||
acquired_at = acquired_at + INTERVAL '1 millisecond',
|
||||
ttl_expires_at = NOW() + INTERVAL '5 minutes',
|
||||
last_refreshed_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[lockId],
|
||||
);
|
||||
}
|
||||
|
||||
describe('fenced refresh/release (D5.10)', () => {
|
||||
test('handle carries its acquisition fence and refresh() returns true while owned', async () => {
|
||||
const handle = await tryAcquireDbLock(engine, 'test-fence-own', 5);
|
||||
expect(handle).not.toBeNull();
|
||||
expect(typeof handle!.acquiredAt).toBe('string');
|
||||
expect(handle!.acquiredAt.length).toBeGreaterThan(0);
|
||||
expect(await handle!.refresh()).toBe(true);
|
||||
// Refresh must not change the acquisition identity.
|
||||
expect(await handle!.refresh()).toBe(true);
|
||||
await handle!.release();
|
||||
});
|
||||
|
||||
test('after a steal, the old handle refresh() returns false and cannot re-take the row', async () => {
|
||||
const handle = await tryAcquireDbLock(engine, 'test-fence-steal', 5);
|
||||
expect(handle).not.toBeNull();
|
||||
await stealRow('test-fence-steal');
|
||||
|
||||
expect(await handle!.refresh()).toBe(false);
|
||||
// The failed refresh must not have bumped the successor's TTL under the
|
||||
// OLD identity — i.e. the row still exists and refresh stays false.
|
||||
expect(await handle!.refresh()).toBe(false);
|
||||
});
|
||||
|
||||
test("release() after a steal is a fenced no-op — the successor's row survives", async () => {
|
||||
const handle = await tryAcquireDbLock(engine, 'test-fence-release', 5);
|
||||
expect(handle).not.toBeNull();
|
||||
await stealRow('test-fence-release');
|
||||
|
||||
await handle!.release(); // must NOT delete the successor's row
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM gbrain_cycle_locks WHERE id = $1`,
|
||||
['test-fence-release'],
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
});
|
||||
|
||||
test('release() while still owned deletes the row (normal path unchanged)', async () => {
|
||||
const handle = await tryAcquireDbLock(engine, 'test-fence-normal', 5);
|
||||
expect(handle).not.toBeNull();
|
||||
await handle!.release();
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM gbrain_cycle_locks WHERE id = $1`,
|
||||
['test-fence-normal'],
|
||||
);
|
||||
expect(rows.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('startCycleLockRefresher (Tier-1 #1 + D5.11)', () => {
|
||||
const fakeLock = (impl: () => Promise<boolean>): LockHandle => ({
|
||||
refresh: impl,
|
||||
release: async () => {},
|
||||
});
|
||||
|
||||
test('aborts the controller with LockStolenError when a fenced refresh returns false', async () => {
|
||||
const controller = new AbortController();
|
||||
const stop = startCycleLockRefresher(fakeLock(async () => false), controller, 'test-lock', 15);
|
||||
try {
|
||||
// Poll instead of a fixed sleep: under full-suite shard load, timer
|
||||
// ticks can be starved well past the nominal interval.
|
||||
const deadline = Date.now() + 5_000;
|
||||
while (!controller.signal.aborted && Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 25));
|
||||
}
|
||||
expect(controller.signal.aborted).toBe(true);
|
||||
expect(controller.signal.reason).toBeInstanceOf(LockStolenError);
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('serializes refreshes — a slow refresh never overlaps the next tick', async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const controller = new AbortController();
|
||||
const stop = startCycleLockRefresher(fakeLock(async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 60));
|
||||
inFlight--;
|
||||
return true;
|
||||
}), controller, 'test-lock', 15);
|
||||
try {
|
||||
await new Promise(r => setTimeout(r, 250));
|
||||
expect(maxInFlight).toBe(1);
|
||||
expect(controller.signal.aborted).toBe(false);
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('a THROWN refresh error is transient — logged, retried, never treated as a steal', async () => {
|
||||
let calls = 0;
|
||||
const controller = new AbortController();
|
||||
const stop = startCycleLockRefresher(fakeLock(async () => {
|
||||
calls++;
|
||||
throw new Error('pooler blip');
|
||||
}), controller, 'test-lock', 15);
|
||||
try {
|
||||
await new Promise(r => setTimeout(r, 120));
|
||||
expect(calls).toBeGreaterThan(1); // kept retrying
|
||||
expect(controller.signal.aborted).toBe(false);
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('stop() halts ticking', async () => {
|
||||
let calls = 0;
|
||||
const controller = new AbortController();
|
||||
const stop = startCycleLockRefresher(fakeLock(async () => { calls++; return true; }), controller, 'test-lock', 15);
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
stop();
|
||||
const after = calls;
|
||||
await new Promise(r => setTimeout(r, 60));
|
||||
expect(calls).toBe(after);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildYieldDuringPhase steal reporting', () => {
|
||||
test('invokes onStolen when refresh() returns false, still fires the outer hook', async () => {
|
||||
let stolenErr: LockStolenError | undefined;
|
||||
let outerRan = false;
|
||||
const fn = buildYieldDuringPhase(
|
||||
{ refresh: async () => false, release: async () => {} },
|
||||
async () => { outerRan = true; },
|
||||
(e) => { stolenErr = e; },
|
||||
);
|
||||
await fn!();
|
||||
expect(stolenErr).toBeInstanceOf(LockStolenError);
|
||||
expect(outerRan).toBe(true);
|
||||
});
|
||||
|
||||
test('does NOT invoke onStolen on a thrown (transient) refresh error', async () => {
|
||||
let stolen = false;
|
||||
const fn = buildYieldDuringPhase(
|
||||
{ refresh: async () => { throw new Error('blip'); }, release: async () => {} },
|
||||
undefined,
|
||||
() => { stolen = true; },
|
||||
);
|
||||
await fn!();
|
||||
expect(stolen).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #3, CONFIRMED) — re-embed upserts must carry
|
||||
* `modality`, or image chunks flip to modality='text' and the image search
|
||||
* arm (filter cc.modality='image') silently returns nothing.
|
||||
*
|
||||
* Pins two layers:
|
||||
* 1. carryChunkMetadata (the now-single shared field list) carries modality
|
||||
* + all 8 code-metadata fields.
|
||||
* 2. The write-side contract that makes the carry load-bearing: upsertChunks
|
||||
* OVERWRITES modality from EXCLUDED, so a merge built via
|
||||
* carryChunkMetadata round-trips an image chunk intact, while a merge
|
||||
* that omits the field (the pre-fix CLI behavior) demonstrably resets it.
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { carryChunkMetadata } from '../src/core/embed-stale.ts';
|
||||
import type { ChunkInput } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
test('carryChunkMetadata carries modality and all code-metadata fields', () => {
|
||||
const loaded = {
|
||||
modality: 'image' as const,
|
||||
language: 'ts',
|
||||
symbol_name: 'fn',
|
||||
symbol_type: 'function',
|
||||
start_line: 1,
|
||||
end_line: 10,
|
||||
parent_symbol_path: ['Mod'],
|
||||
doc_comment: 'doc',
|
||||
symbol_name_qualified: 'Mod.fn',
|
||||
};
|
||||
const base: ChunkInput = { chunk_index: 0, chunk_text: 'x', chunk_source: 'compiled_truth' };
|
||||
const out = carryChunkMetadata(loaded, base);
|
||||
expect(out.modality).toBe('image');
|
||||
expect(out.language).toBe('ts');
|
||||
expect(out.symbol_name_qualified).toBe('Mod.fn');
|
||||
// Absent fields stay undefined, never null-through (DB rows can carry null
|
||||
// at runtime; the ?? undefined in the carry normalizes them).
|
||||
const sparse = carryChunkMetadata({}, base);
|
||||
expect(sparse.modality).toBeUndefined();
|
||||
});
|
||||
|
||||
test('re-embed merge via carryChunkMetadata keeps an image chunk image', async () => {
|
||||
await engine.putPage('img-page', { type: 'note', title: 'img', compiled_truth: 'ocr text here', frontmatter: {} });
|
||||
await engine.upsertChunks('img-page', [
|
||||
{ chunk_index: 0, chunk_text: 'ocr text here', chunk_source: 'compiled_truth', modality: 'image' },
|
||||
]);
|
||||
|
||||
// Simulate the embed --stale merge: load chunks, rebuild ChunkInputs, upsert.
|
||||
const loaded = await engine.getChunks('img-page');
|
||||
expect(loaded[0]!.modality).toBe('image');
|
||||
const merged = loaded.map(c => carryChunkMetadata(c, {
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
token_count: c.token_count || 1,
|
||||
}));
|
||||
await engine.upsertChunks('img-page', merged);
|
||||
|
||||
const after = await engine.getChunks('img-page');
|
||||
expect(after[0]!.modality).toBe('image');
|
||||
});
|
||||
|
||||
test('write-side contract: omitting modality (pre-fix behavior) resets it to text', async () => {
|
||||
await engine.putPage('img-page-2', { type: 'note', title: 'img2', compiled_truth: 'more ocr', frontmatter: {} });
|
||||
await engine.upsertChunks('img-page-2', [
|
||||
{ chunk_index: 0, chunk_text: 'more ocr', chunk_source: 'compiled_truth', modality: 'image' },
|
||||
]);
|
||||
// The pre-fix CLI merge: field list without modality.
|
||||
await engine.upsertChunks('img-page-2', [
|
||||
{ chunk_index: 0, chunk_text: 'more ocr', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
const after = await engine.getChunks('img-page-2');
|
||||
// This assertion documents WHY the carry is load-bearing: the upsert
|
||||
// overwrites, so the omission class corrupts. If this ever flips to
|
||||
// COALESCE semantics, the carry (and this test) can be revisited.
|
||||
expect(after[0]!.modality).toBe('text');
|
||||
});
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
import { test, expect, describe, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import {
|
||||
readContentChunksEmbeddingDim,
|
||||
embeddingMismatchMessage,
|
||||
@@ -73,8 +74,12 @@ describe('readContentChunksEmbeddingDim', () => {
|
||||
test('returns { exists: false, dims: null } on a fresh brain (no initSchema)', async () => {
|
||||
// One-off engine for the fresh-brain case. Never call initSchema so
|
||||
// content_chunks doesn't exist yet. Cleaned up at end of test.
|
||||
// W0: the default-on snapshot loads a fully-migrated schema at connect,
|
||||
// which breaks this test's truly-empty-DB premise — opt out for this boot.
|
||||
const fresh = new PGLiteEngine();
|
||||
await fresh.connect({});
|
||||
await withEnv({ GBRAIN_PGLITE_SNAPSHOT: undefined }, async () => {
|
||||
await fresh.connect({});
|
||||
});
|
||||
try {
|
||||
const result = await readContentChunksEmbeddingDim(fresh);
|
||||
expect(result.exists).toBe(false);
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
// Guard self-test fixture (known-BAD): positional $N::jsonb + JSON.stringify
|
||||
// in the same call span — the #2339 shape that aborted every sync.
|
||||
declare const engine: { executeRaw: (sql: string, params?: unknown[]) => Promise<unknown[]> };
|
||||
declare const x: Record<string, unknown>;
|
||||
export async function bad(): Promise<void> {
|
||||
await engine.executeRaw(`UPDATE op_checkpoints SET pin = $1::jsonb WHERE id = $2`, [JSON.stringify(x), 1]);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Guard self-test fixture (known-GOOD): the text-hop positional spelling.
|
||||
declare const engine: { executeRaw: (sql: string, params?: unknown[]) => Promise<unknown[]> };
|
||||
declare const x: Record<string, unknown>;
|
||||
export async function good(): Promise<void> {
|
||||
await engine.executeRaw(`UPDATE op_checkpoints SET pin = $1::text::jsonb WHERE id = $2`, [JSON.stringify(x), 1]);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// Guard self-test fixture (known-BAD): template-interpolated stringify into a
|
||||
// direct ::jsonb cast, WITH nested parens (the pre-W0 regex hole).
|
||||
declare const sql: (strings: TemplateStringsArray, ...vals: unknown[]) => Promise<unknown>;
|
||||
declare const obj: { get: () => unknown };
|
||||
export async function bad(): Promise<void> {
|
||||
await sql`UPDATE pages SET frontmatter = ${JSON.stringify(obj.get())}::jsonb WHERE id = 1`;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Guard self-test fixture (known-GOOD): the safe text-hop spelling — binds as
|
||||
// text, the cast parses it (no direct )}::jsonb adjacency).
|
||||
declare const sql: (strings: TemplateStringsArray, ...vals: unknown[]) => Promise<unknown>;
|
||||
declare const obj: { get: () => unknown };
|
||||
export async function good(): Promise<void> {
|
||||
await sql`UPDATE pages SET frontmatter = ${JSON.stringify(obj.get())}::text::jsonb WHERE id = 1`;
|
||||
}
|
||||
|
||||
// Multi-interpolation line: a SAFE ::text::jsonb stringify followed by a
|
||||
// separate paren-bearing non-stringify ::jsonb interpolation on the SAME
|
||||
// line. A greedy argument matcher spans from stringify( to the second
|
||||
// interpolation's `)}::jsonb` and false-positives; the bracket-bounded
|
||||
// pattern must not (ship-review edge case, proven by repro).
|
||||
declare function getRaw(): unknown;
|
||||
export async function goodMultiInterpolation(): Promise<void> {
|
||||
await sql`UPDATE pages SET a = ${JSON.stringify(obj.get())}::text::jsonb, b = ${getRaw()}::jsonb WHERE id = 2`;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Guard self-test fixture (known-BAD, multi-line): withRetry( on one line and
|
||||
// the engine batch call two lines later — exercises the perl window pass
|
||||
// (the single-line grep cannot see this shape).
|
||||
declare const engine: { upsertChunks: (slug: string, chunks: unknown[]) => Promise<void> };
|
||||
declare function withRetry<T>(fn: () => Promise<T>, opts?: unknown): Promise<T>;
|
||||
declare const chunks: unknown[];
|
||||
export async function badMultiline(): Promise<void> {
|
||||
await withRetry(
|
||||
async () =>
|
||||
engine.upsertChunks('slug', chunks),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Guard self-test fixture (known-BAD): the CANONICAL banned shape — the exact
|
||||
// arrow-fn form the pre-W0 regex could not match ([^)]* stopped at `() =>`).
|
||||
declare const engine: { addLinksBatch: (rows: unknown[]) => Promise<void> };
|
||||
declare function withRetry<T>(fn: () => Promise<T>, opts?: unknown): Promise<T>;
|
||||
declare const rows: unknown[];
|
||||
declare const BULK_RETRY_OPTS: unknown;
|
||||
export async function bad(): Promise<void> {
|
||||
await withRetry(() => engine.addLinksBatch(rows), BULK_RETRY_OPTS);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Guard self-test fixture (known-GOOD): direct engine call, no outer retry.
|
||||
declare const engine: { addLinksBatch: (rows: unknown[], opts?: unknown) => Promise<void> };
|
||||
declare const rows: unknown[];
|
||||
export async function good(): Promise<void> {
|
||||
await engine.addLinksBatch(rows, { auditSite: 'fixture.good' });
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* The pinned test-suite embedding shape (OpenAI legacy 1536-d).
|
||||
*
|
||||
* ONE definition, two consumers (W0 fix-wave — no hand-copied twins):
|
||||
* - test/helpers/legacy-embedding-preload.ts (bunfig preload) pins the
|
||||
* gateway to this shape for every `bun test` file.
|
||||
* - scripts/build-pglite-snapshot.ts builds the snapshot fixture under the
|
||||
* SAME shape, so the baked vector(dims) columns match what the pinned
|
||||
* tests write. (The W0 incident: the build script ran with an
|
||||
* unconfigured gateway → shipped-default 1280-d columns → every
|
||||
* 1536-d-writing test failed with "expected 1280 dimensions, not 1536"
|
||||
* the moment the snapshot became default-on.)
|
||||
*/
|
||||
export const LEGACY_EMBEDDING_CONFIG = {
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
} as const;
|
||||
@@ -25,10 +25,9 @@ import {
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { beforeEach } from 'bun:test';
|
||||
|
||||
const LEGACY_CONFIG = {
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
} as const;
|
||||
// W0 fix-wave: shared with scripts/build-pglite-snapshot.ts so the snapshot
|
||||
// fixture is baked under the exact shape this preload pins.
|
||||
import { LEGACY_EMBEDDING_CONFIG as LEGACY_CONFIG } from './legacy-embedding-config.ts';
|
||||
|
||||
function legacyGatewayConfig() {
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #5) — runImport must THROW a typed ImportAbortError on
|
||||
* preflight/argv failures, never process.exit(1).
|
||||
*
|
||||
* runImport is invoked in-process by the sync_brain MCP op (performFullSync),
|
||||
* the autopilot daemon, and the minion sync handler. Pre-fix, a first sync
|
||||
* against a brain with unusable embedding credentials TERMINATED the calling
|
||||
* process — the stdio MCP server just vanished mid-tool-call. The CLI dispatch
|
||||
* site maps the typed error back to exit(1), keeping CLI behavior identical.
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { runImport, ImportAbortError } from '../src/commands/import.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
// The abort sites fire before any WRITE; the preamble runs a couple of
|
||||
// read-only lookups (sources, config), so the stub answers those with
|
||||
// empty sets.
|
||||
const engineStub = {
|
||||
kind: 'pglite',
|
||||
executeRaw: async () => [],
|
||||
getConfig: async () => null,
|
||||
} as unknown as BrainEngine;
|
||||
|
||||
async function expectAbort(args: string[], reasonFragment: string): Promise<void> {
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await runImport(engineStub, args);
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(ImportAbortError);
|
||||
const err = thrown as ImportAbortError;
|
||||
expect(err.exitCode).toBe(1);
|
||||
expect(err.alreadyReported).toBe(true);
|
||||
expect(err.message).toContain(reasonFragment);
|
||||
}
|
||||
|
||||
test('missing dir arg → typed abort, not process death', async () => {
|
||||
await expectAbort(['--no-embed'], 'no import directory');
|
||||
});
|
||||
|
||||
test('invalid --workers → typed abort', async () => {
|
||||
await expectAbort(['--no-embed', '--workers', '0', '/tmp'], 'invalid --workers');
|
||||
});
|
||||
|
||||
test('unreadable import target → typed abort', async () => {
|
||||
await expectAbort(['--no-embed', '/definitely/not/a/real/dir-w0-test'], 'not readable');
|
||||
});
|
||||
|
||||
test('the calling process survives the abort (the actual Tier-1 bug)', async () => {
|
||||
// Trivially true if we got here after three aborts above, but assert it
|
||||
// explicitly: the process is alive and can keep dispatching.
|
||||
expect(process.pid).toBeGreaterThan(0);
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #14) — `gbrain lint --fix` scans once and reports the
|
||||
* TRUE fixed count.
|
||||
*
|
||||
* Pre-fix, runLint ran its own read+lint+fix loop for human detail, then
|
||||
* called runLintCore a second time for the summary — every page linted
|
||||
* twice, and because the first pass had already written the fixes, the
|
||||
* summary's total_fixed counted fixes against already-fixed content: the
|
||||
* CLI printed "0 auto-fixed" after fixing N issues.
|
||||
*/
|
||||
|
||||
import { test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { runLintCore } from '../src/commands/lint.ts';
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'gbrain-lint-w0-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('single pass: total_fixed reports the fixes THIS run applied', async () => {
|
||||
// A whole-file ```markdown wrapper is a fixable rule (code-fence-wrap).
|
||||
writeFileSync(join(dir, 'wrapped.md'), '```markdown\n# Hello\n\nBody text.\n```\n');
|
||||
writeFileSync(join(dir, 'clean.md'), '# Clean\n\nNothing to fix here.\n');
|
||||
|
||||
const scanned: string[] = [];
|
||||
const reported: Array<{ relPath: string; fixedCount: number }> = [];
|
||||
const result = await runLintCore({
|
||||
target: dir,
|
||||
fix: true,
|
||||
contentSanity: { disabled: true } as never,
|
||||
onPageScanned: () => scanned.push('tick'),
|
||||
onPageIssues: (relPath, _issues, fixedCount) => reported.push({ relPath, fixedCount }),
|
||||
});
|
||||
|
||||
// The bug: this was 0 after fixing. It must equal the fixes applied NOW.
|
||||
expect(result.total_fixed).toBeGreaterThan(0);
|
||||
// Each page scanned exactly once (2 pages, 2 ticks — not 4).
|
||||
expect(scanned.length).toBe(2);
|
||||
// The per-page hook carried the real fixed count for the fixed page
|
||||
// (clean.md may still surface non-fixable advisory issues — irrelevant here).
|
||||
const wrapped = reported.find(r => r.relPath === 'wrapped.md');
|
||||
expect(wrapped).toBeDefined();
|
||||
expect(wrapped!.fixedCount).toBeGreaterThan(0);
|
||||
expect(result.total_fixed).toBe(wrapped!.fixedCount);
|
||||
// And the fix actually landed on disk.
|
||||
expect(readFileSync(join(dir, 'wrapped.md'), 'utf-8').startsWith('```')).toBe(false);
|
||||
|
||||
// Second run: nothing left to fix — counts stay honest.
|
||||
const again = await runLintCore({ target: dir, fix: true, contentSanity: { disabled: true } as never });
|
||||
expect(again.total_fixed).toBe(0);
|
||||
});
|
||||
@@ -50,8 +50,13 @@ describe('hasPendingMigrations', () => {
|
||||
}, 30000);
|
||||
|
||||
test('returns true when version config is missing entirely (defensive default)', async () => {
|
||||
// W0: opt out of the default-on snapshot — this test's premise is an
|
||||
// empty PGlite with no config table at all.
|
||||
const priorSnapshot = process.env.GBRAIN_PGLITE_SNAPSHOT;
|
||||
delete process.env.GBRAIN_PGLITE_SNAPSHOT;
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
if (priorSnapshot !== undefined) process.env.GBRAIN_PGLITE_SNAPSHOT = priorSnapshot;
|
||||
try {
|
||||
// Don't call initSchema. Probe against an empty PGlite — getConfig should
|
||||
// either return null (treated as version=1) or throw on missing config
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #4) — a child dead-lettered via max-stall must notify
|
||||
* and unblock its aggregator parent, exactly like timeout/failure kills do.
|
||||
*
|
||||
* Pre-fix, handleStalled's dead-letter branch emitted no child_done and never
|
||||
* flipped the parent out of 'waiting-children': the parent hung forever
|
||||
* unless it happened to carry its own timeout_ms (self-heal-proof — verified
|
||||
* in adversarial review: resolveParent has no periodic caller and the worker
|
||||
* only logged counts).
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
import type { ChildDoneMessage } from '../src/core/minions/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let queue: MinionQueue;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
queue = new MinionQueue(engine);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw('DELETE FROM minion_inbox');
|
||||
await engine.executeRaw('DELETE FROM minion_jobs');
|
||||
});
|
||||
|
||||
async function jobStatus(id: number): Promise<string> {
|
||||
const rows = await engine.executeRaw<{ status: string }>(
|
||||
`SELECT status FROM minion_jobs WHERE id = $1`, [id],
|
||||
);
|
||||
return rows[0]!.status;
|
||||
}
|
||||
|
||||
test('stall-exhausted child → child_done(dead) in parent inbox + parent unblocked', async () => {
|
||||
const parent = await queue.add('aggregator', {});
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [parent.id],
|
||||
);
|
||||
const child = await queue.add('child', {}, { parent_job_id: parent.id, max_stalled: 1 });
|
||||
|
||||
// Claim the child, then expire its lock so the stall sweep sees it; with
|
||||
// max_stalled=1 the first stall dead-letters it.
|
||||
const claimed = await queue.claim('tok-stall', 30_000, 'default', ['child', 'aggregator']);
|
||||
expect(claimed?.id).toBe(child.id);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET lock_until = now() - interval '1 second' WHERE id = $1`, [child.id],
|
||||
);
|
||||
|
||||
const { requeued, dead } = await queue.handleStalled();
|
||||
expect(requeued).toHaveLength(0);
|
||||
expect(dead.map(j => j.id)).toEqual([child.id]);
|
||||
|
||||
// The fix, part 1: parent got child_done with outcome 'dead'.
|
||||
const inbox = await engine.executeRaw<{ payload: unknown }>(
|
||||
`SELECT payload FROM minion_inbox WHERE job_id = $1`, [parent.id],
|
||||
);
|
||||
expect(inbox).toHaveLength(1);
|
||||
const msg = (typeof inbox[0]!.payload === 'string'
|
||||
? JSON.parse(inbox[0]!.payload as string)
|
||||
: inbox[0]!.payload) as ChildDoneMessage;
|
||||
expect(msg.type).toBe('child_done');
|
||||
expect(msg.child_id).toBe(child.id);
|
||||
expect(msg.outcome).toBe('dead');
|
||||
expect(msg.error).toBe('max stalled count exceeded');
|
||||
|
||||
// The fix, part 2: parent flipped out of waiting-children.
|
||||
expect(await jobStatus(parent.id)).toBe('waiting');
|
||||
});
|
||||
|
||||
test('stall-requeued child (budget left) does NOT touch the parent', async () => {
|
||||
const parent = await queue.add('aggregator', {});
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [parent.id],
|
||||
);
|
||||
const child = await queue.add('child', {}, { parent_job_id: parent.id, max_stalled: 5 });
|
||||
const claimed = await queue.claim('tok-stall2', 30_000, 'default', ['child', 'aggregator']);
|
||||
expect(claimed?.id).toBe(child.id);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET lock_until = now() - interval '1 second' WHERE id = $1`, [child.id],
|
||||
);
|
||||
|
||||
const { requeued, dead } = await queue.handleStalled();
|
||||
expect(dead).toHaveLength(0);
|
||||
expect(requeued.map(j => j.id)).toEqual([child.id]);
|
||||
expect(await jobStatus(child.id)).toBe('waiting');
|
||||
|
||||
const inbox = await engine.executeRaw<{ payload: unknown }>(
|
||||
`SELECT payload FROM minion_inbox WHERE job_id = $1`, [parent.id],
|
||||
);
|
||||
expect(inbox).toHaveLength(0);
|
||||
expect(await jobStatus(parent.id)).toBe('waiting-children');
|
||||
});
|
||||
|
||||
test('all three reapers route through the shared kill tail with their own outcome/error (D5.5)', async () => {
|
||||
// handleTimeouts → outcome 'timeout', error 'timeout exceeded'
|
||||
const p1 = await queue.add('agg1', {});
|
||||
await engine.executeRaw(`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [p1.id]);
|
||||
const c1 = await queue.add('c1', {}, { parent_job_id: p1.id });
|
||||
await queue.claim('t1', 30_000, 'default', ['c1', 'agg1']);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET timeout_at = now() - interval '1 second', lock_until = now() + interval '1 minute' WHERE id = $1`, [c1.id],
|
||||
);
|
||||
await queue.handleTimeouts();
|
||||
|
||||
// handleWallClockTimeouts → outcome 'timeout', error 'wall-clock timeout exceeded'
|
||||
const p2 = await queue.add('agg2', {});
|
||||
await engine.executeRaw(`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [p2.id]);
|
||||
const c2 = await queue.add('c2', {}, { parent_job_id: p2.id });
|
||||
await queue.claim('t2', 30_000, 'default', ['c2', 'agg2']);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET started_at = now() - interval '2 hours' WHERE id = $1`, [c2.id],
|
||||
);
|
||||
await queue.handleWallClockTimeouts(30_000);
|
||||
|
||||
const expectMsg = async (parentId: number, childId: number, outcome: string, error: string) => {
|
||||
const inbox = await engine.executeRaw<{ payload: unknown }>(
|
||||
`SELECT payload FROM minion_inbox WHERE job_id = $1`, [parentId],
|
||||
);
|
||||
expect(inbox).toHaveLength(1);
|
||||
const msg = (typeof inbox[0]!.payload === 'string'
|
||||
? JSON.parse(inbox[0]!.payload as string)
|
||||
: inbox[0]!.payload) as ChildDoneMessage;
|
||||
expect(msg.child_id).toBe(childId);
|
||||
expect(msg.outcome).toBe(outcome as ChildDoneMessage['outcome']);
|
||||
expect(msg.error).toBe(error);
|
||||
expect(await jobStatus(parentId)).toBe('waiting');
|
||||
};
|
||||
await expectMsg(p1.id, c1.id, 'timeout', 'timeout exceeded');
|
||||
await expectMsg(p2.id, c2.id, 'timeout', 'wall-clock timeout exceeded');
|
||||
});
|
||||
|
||||
test('retroactive sweep: a parent stranded BEFORE the fix (children already dead) self-heals on the next stall tick', async () => {
|
||||
// Simulate the pre-upgrade world: child is already terminal ('dead') with
|
||||
// NO child_done ever emitted, parent parked in waiting-children. No
|
||||
// per-kill unblock will ever revisit this pair — only the sweep can.
|
||||
const parent = await queue.add('agg-stranded', {});
|
||||
const child = await queue.add('child-stranded', {}, { parent_job_id: parent.id });
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [parent.id],
|
||||
);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'dead', error_text = 'max stalled count exceeded (pre-upgrade)' WHERE id = $1`, [child.id],
|
||||
);
|
||||
|
||||
const { requeued, dead } = await queue.handleStalled();
|
||||
expect(requeued).toHaveLength(0);
|
||||
expect(dead).toHaveLength(0);
|
||||
// The sweep healed the stranded parent even though THIS tick killed nothing.
|
||||
expect(await jobStatus(parent.id)).toBe('waiting');
|
||||
});
|
||||
|
||||
test('retroactive sweep does NOT unblock a parent with a live child', async () => {
|
||||
const parent = await queue.add('agg-live', {});
|
||||
await queue.add('child-live', {}, { parent_job_id: parent.id });
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting-children' WHERE id = $1`, [parent.id],
|
||||
);
|
||||
await queue.handleStalled();
|
||||
expect(await jobStatus(parent.id)).toBe('waiting-children');
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* W0 fix-wave (Tier-1 #7) — started_at is reset on every AUTOMATIC re-run
|
||||
* path, so handleWallClockTimeouts (anchored on now() - started_at) measures
|
||||
* per-attempt runtime, never time a job spent parked in backoff.
|
||||
*
|
||||
* Pre-fix, only the manual `jobs retry` path cleared started_at (its
|
||||
* docstring documented the bug); the four automatic paths — failJob's
|
||||
* delayed branch, handleStalled's requeue, promoteDelayed, and
|
||||
* releaseLeaseFullJob (the 4th site, found in adversarial verification) —
|
||||
* all preserved the FIRST claim's timestamp, so an exponential-backoff job
|
||||
* could be dead-lettered by the wall-clock sweep before executing a single
|
||||
* line of its retry attempt.
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let queue: MinionQueue;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
queue = new MinionQueue(engine);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw('DELETE FROM minion_jobs');
|
||||
});
|
||||
|
||||
async function jobStatus(id: number): Promise<string> {
|
||||
const rows = await engine.executeRaw<{ status: string }>(
|
||||
`SELECT status FROM minion_jobs WHERE id = $1`, [id],
|
||||
);
|
||||
return rows[0]!.status;
|
||||
}
|
||||
|
||||
async function startedAtOf(id: number): Promise<string | null> {
|
||||
const rows = await engine.executeRaw<{ started_at: string | null }>(
|
||||
`SELECT started_at::text AS started_at FROM minion_jobs WHERE id = $1`, [id],
|
||||
);
|
||||
return rows[0]?.started_at ?? null;
|
||||
}
|
||||
|
||||
async function addAndClaim(name = 'noop'): Promise<{ id: number; token: string }> {
|
||||
const job = await queue.add(name, {});
|
||||
const token = `tok-${Math.random().toString(36).slice(2)}`;
|
||||
const claimed = await queue.claim(token, 30_000, 'default', [name]);
|
||||
expect(claimed?.id).toBe(job.id);
|
||||
expect(await startedAtOf(job.id)).not.toBeNull();
|
||||
return { id: job.id, token };
|
||||
}
|
||||
|
||||
test("failJob's delayed branch clears started_at (terminal branches keep it)", async () => {
|
||||
const { id, token } = await addAndClaim();
|
||||
const delayed = await queue.failJob(id, token, 'boom', 'delayed', 50);
|
||||
expect(delayed?.status).toBe('delayed');
|
||||
expect(await startedAtOf(id)).toBeNull();
|
||||
|
||||
// Terminal failure keeps started_at (it feeds duration accounting).
|
||||
const { id: id2, token: token2 } = await addAndClaim();
|
||||
const dead = await queue.failJob(id2, token2, 'boom', 'dead');
|
||||
expect(dead?.status).toBe('dead');
|
||||
expect(await startedAtOf(id2)).not.toBeNull();
|
||||
});
|
||||
|
||||
test("handleStalled's requeue branch clears started_at", async () => {
|
||||
const { id } = await addAndClaim();
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET lock_until = now() - interval '1 second' WHERE id = $1`, [id],
|
||||
);
|
||||
const { requeued } = await queue.handleStalled();
|
||||
expect(requeued.map(j => j.id)).toContain(id);
|
||||
expect(await startedAtOf(id)).toBeNull();
|
||||
});
|
||||
|
||||
test('releaseLeaseFullJob clears started_at (lease bounce burns no wall-clock)', async () => {
|
||||
const { id, token } = await addAndClaim();
|
||||
const released = await queue.releaseLeaseFullJob(id, token, 'rate lease full (8/8)', 50);
|
||||
expect(released?.status).toBe('delayed');
|
||||
expect(await startedAtOf(id)).toBeNull();
|
||||
});
|
||||
|
||||
test('promoteDelayed defensively clears any stale started_at', async () => {
|
||||
const job = await queue.add('noop', {});
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs
|
||||
SET status = 'delayed', delay_until = now() - interval '1 second',
|
||||
started_at = now() - interval '1 hour'
|
||||
WHERE id = $1`, [job.id],
|
||||
);
|
||||
const promoted = await queue.promoteDelayed();
|
||||
expect(promoted.map(j => j.id)).toContain(job.id);
|
||||
expect(await startedAtOf(job.id)).toBeNull();
|
||||
});
|
||||
|
||||
test('end-to-end: a retried job survives the wall-clock sweep on its fresh attempt', async () => {
|
||||
const { id, token } = await addAndClaim();
|
||||
// Simulate a long first attempt: backdate started_at far past the sweep
|
||||
// threshold (no timeout_ms → threshold = lockDurationMs * 2 * max_stalled).
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET started_at = now() - interval '1 hour' WHERE id = $1`, [id],
|
||||
);
|
||||
await queue.failJob(id, token, 'first attempt failed', 'delayed', 0);
|
||||
const promoted = await queue.promoteDelayed();
|
||||
expect(promoted.map(j => j.id)).toContain(id);
|
||||
|
||||
const token2 = 'tok-second-attempt';
|
||||
const reclaimed = await queue.claim(token2, 30_000, 'default', ['noop']);
|
||||
expect(reclaimed?.id).toBe(id);
|
||||
|
||||
// Pre-fix: started_at still said now()-1h → 3.6M ms > 30_000*2*max_stalled
|
||||
// and the very next sweep dead-lettered the fresh attempt. Post-fix the
|
||||
// re-claim stamped a fresh started_at, so the sweep spares it.
|
||||
const killed = await queue.handleWallClockTimeouts(30_000);
|
||||
expect(killed.map(j => j.id)).not.toContain(id);
|
||||
|
||||
// Negative control — the sweep itself still works: backdate the ACTIVE
|
||||
// attempt past the threshold and it dies.
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET started_at = now() - interval '1 hour' WHERE id = $1`, [id],
|
||||
);
|
||||
const killed2 = await queue.handleWallClockTimeouts(30_000);
|
||||
expect(killed2.map(j => j.id)).toContain(id);
|
||||
});
|
||||
|
||||
test('red-team 5th path: a re-claimed aggregator parent survives the wall-clock sweep', async () => {
|
||||
// Parent parked in waiting-children for LONGER than the wall-clock
|
||||
// threshold while its child runs; every unblock path (killJobs,
|
||||
// completeJob resolve, failJob branches, cancelJob, retroactive sweep,
|
||||
// resolveParent) must clear started_at, or claim()'s COALESCE keeps the
|
||||
// attempt-1 anchor and the sweep dead-letters the aggregation attempt
|
||||
// before it executes a line.
|
||||
const parent = await queue.add('agg-wallclock', {});
|
||||
const child = await queue.add('child-wc', {}, { parent_job_id: parent.id, max_stalled: 1 });
|
||||
// Parent claimed long ago (attempt-1 anchor), then parked waiting-children.
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET status = 'waiting-children', started_at = now() - interval '1 hour' WHERE id = $1`, [parent.id],
|
||||
);
|
||||
// Child stalls to death → killJobs unblocks the parent.
|
||||
await queue.claim('tok-wc', 30_000, 'default', ['child-wc', 'agg-wallclock']);
|
||||
await engine.executeRaw(
|
||||
`UPDATE minion_jobs SET lock_until = now() - interval '1 second' WHERE id = $1`, [child.id],
|
||||
);
|
||||
await queue.handleStalled();
|
||||
expect(await jobStatus(parent.id)).toBe('waiting');
|
||||
// The fix: unblock cleared the stale anchor.
|
||||
expect(await startedAtOf(parent.id)).toBeNull();
|
||||
|
||||
// Parent re-claims for its aggregation attempt → fresh anchor → survives.
|
||||
const reclaimed = await queue.claim('tok-wc2', 30_000, 'default', ['agg-wallclock', 'child-wc']);
|
||||
expect(reclaimed?.id).toBe(parent.id);
|
||||
const killed = await queue.handleWallClockTimeouts(30_000);
|
||||
expect(killed.map(j => j.id)).not.toContain(parent.id);
|
||||
});
|
||||
@@ -304,16 +304,21 @@ describe('check-engine-dynamic-import.sh', () => {
|
||||
});
|
||||
|
||||
describe('engine dynamic-import guard wiring', () => {
|
||||
it('is invoked through bash by check:all', () => {
|
||||
it('is wired into the verify registry (W0: check:all deleted; CHECKS array is THE registry)', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf8')) as {
|
||||
scripts: Record<string, string>;
|
||||
};
|
||||
expect(pkg.scripts['check:engine-dynamic-import']).toBe(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
expect(pkg.scripts['check:all']).toContain(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
// The stale duplicate registry is gone for good…
|
||||
expect(pkg.scripts['check:all']).toBeUndefined();
|
||||
// …and the single registry carries this guard.
|
||||
const checks = readFileSync(
|
||||
new URL('../../scripts/run-verify-parallel.sh', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
expect(checks).toContain('"check:engine-dynamic-import"');
|
||||
});
|
||||
|
||||
it('is listed by the authoritative verify dispatcher', () => {
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* W0 ship-review coverage (GAP-2) — the snapshot loader's shape + hash guards.
|
||||
*
|
||||
* The fixture is default-on for every `bun run test`, so a wrong snapshot
|
||||
* poisons the whole suite (the 1280-vs-1536 incident: 115 failures from one
|
||||
* root cause). These tests pin the three refusal paths and the
|
||||
* handler-aware hash (D5.13).
|
||||
*/
|
||||
|
||||
import { test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { tryLoadSnapshot, computeSnapshotSchemaHash } from '../src/core/pglite-engine.ts';
|
||||
import { MIGRATIONS } from '../src/core/migrate.ts';
|
||||
import { PGLITE_SCHEMA_SQL } from '../src/core/pglite-schema.ts';
|
||||
import { getEmbeddingDimensions, getEmbeddingModel } from '../src/core/ai/gateway.ts';
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'gbrain-snap-guard-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function writeFixture(versionContent: string): string {
|
||||
const tarPath = join(dir, 'snap.tar');
|
||||
writeFileSync(tarPath, 'not-a-real-tar-but-existence-is-what-matters');
|
||||
writeFileSync(join(dir, 'snap.version'), versionContent);
|
||||
return tarPath;
|
||||
}
|
||||
|
||||
const currentHash = () => computeSnapshotSchemaHash(MIGRATIONS, PGLITE_SCHEMA_SQL, crypto);
|
||||
|
||||
test('pre-W0 hash-only version file (no shape lines) is refused', () => {
|
||||
const tar = writeFixture(`${currentHash()}\n`);
|
||||
expect(tryLoadSnapshot(tar)).toBeNull();
|
||||
});
|
||||
|
||||
test('dims mismatch is refused even with a matching hash', () => {
|
||||
const tar = writeFixture(`${currentHash()}\ndims=99999\nmodel=${getEmbeddingModel()}\n`);
|
||||
expect(tryLoadSnapshot(tar)).toBeNull();
|
||||
});
|
||||
|
||||
test('model mismatch is refused even with a matching hash', () => {
|
||||
const tar = writeFixture(`${currentHash()}\ndims=${getEmbeddingDimensions()}\nmodel=other:model\n`);
|
||||
expect(tryLoadSnapshot(tar)).toBeNull();
|
||||
});
|
||||
|
||||
test('stale schema hash is refused even with a matching shape', () => {
|
||||
const tar = writeFixture(`deadbeef\ndims=${getEmbeddingDimensions()}\nmodel=${getEmbeddingModel()}\n`);
|
||||
expect(tryLoadSnapshot(tar)).toBeNull();
|
||||
});
|
||||
|
||||
test('matching hash + shape loads the blob', () => {
|
||||
const tar = writeFixture(`${currentHash()}\ndims=${getEmbeddingDimensions()}\nmodel=${getEmbeddingModel()}\n`);
|
||||
const blob = tryLoadSnapshot(tar);
|
||||
expect(blob).not.toBeNull();
|
||||
expect(blob!.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('D5.13: a migration handler edit changes the hash (sql-only hashing missed 19 handler migrations)', () => {
|
||||
const base = [
|
||||
{ version: 1, name: 'a', sql: 'CREATE TABLE t(x int)' },
|
||||
{ version: 2, name: 'b', sql: '', handler: async () => 'original' },
|
||||
];
|
||||
const edited = [
|
||||
{ version: 1, name: 'a', sql: 'CREATE TABLE t(x int)' },
|
||||
{ version: 2, name: 'b', sql: '', handler: async () => 'EDITED BODY' },
|
||||
];
|
||||
const h1 = computeSnapshotSchemaHash(base, 'schema', crypto);
|
||||
const h2 = computeSnapshotSchemaHash(edited, 'schema', crypto);
|
||||
expect(h1).not.toBe(h2);
|
||||
// And identical handlers hash identically (determinism).
|
||||
const h3 = computeSnapshotSchemaHash(base, 'schema', crypto);
|
||||
expect(h1).toBe(h3);
|
||||
});
|
||||
@@ -216,6 +216,7 @@ describe('#1849 refresh-failure fails safe (F1A)', () => {
|
||||
let refreshCalls = 0;
|
||||
const failingLock: DbLockHandle = {
|
||||
id: 'x',
|
||||
acquiredAt: 'test-fence',
|
||||
refresh: async () => { refreshCalls++; throw new Error('pooler down'); },
|
||||
release: async () => {},
|
||||
};
|
||||
@@ -244,7 +245,8 @@ describe('#1849 refresh-failure fails safe (F1A)', () => {
|
||||
let mode: 'fail' | 'ok' = 'fail';
|
||||
const flakyLock: DbLockHandle = {
|
||||
id: 'x',
|
||||
refresh: async () => { if (mode === 'fail') throw new Error('blip'); },
|
||||
acquiredAt: 'test-fence',
|
||||
refresh: async () => { if (mode === 'fail') throw new Error('blip'); return true; },
|
||||
release: async () => {},
|
||||
};
|
||||
sup._setDbLockForTests(flakyLock);
|
||||
@@ -264,3 +266,31 @@ describe('#1849 refresh-failure fails safe (F1A)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('W0 (D5.10): fenced refresh returning false is CERTAIN loss — immediate LOCK_LOST, no threshold', async () => {
|
||||
const sup = new MinionSupervisor(engine, { cliPath: '/bin/sh', healthInterval: 0, json: true });
|
||||
const exitSpy = spyOn(process, 'exit').mockImplementation(((_code?: number) => {
|
||||
throw new Error(`exit:${_code}`);
|
||||
}) as never);
|
||||
|
||||
let refreshCalls = 0;
|
||||
const stolenLock: DbLockHandle = {
|
||||
id: 'x',
|
||||
acquiredAt: 'test-fence',
|
||||
refresh: async () => { refreshCalls++; return false; },
|
||||
release: async () => {},
|
||||
};
|
||||
sup._setDbLockForTests(stolenLock);
|
||||
|
||||
try {
|
||||
// FIRST tick exits — a fenced miss is proof of loss, not a blip to
|
||||
// count toward SUPERVISOR_LOCK_REFRESH_MAX_FAILURES (pre-fix it was
|
||||
// treated as SUCCESS and reset the failure counter while a second
|
||||
// supervisor drained the same queue).
|
||||
try { await sup._refreshDbLockForTests(); } catch { /* exit stub throws */ }
|
||||
expect(exitSpy).toHaveBeenCalledWith(ExitCodes.LOCK_LOST);
|
||||
expect(refreshCalls).toBe(1);
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user