mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 09:52:22 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc0504785a |
@@ -61,10 +61,7 @@ jobs:
|
||||
- name: Run JSONB double-encode parity tests on real Postgres
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
# --timeout also raises bun's 5s default hook budget (beforeAll/afterAll
|
||||
# do NOT inherit a test's third-arg timeout; verified on bun 1.3.x).
|
||||
# Every runner script in scripts/ passes it; bare invocations must too.
|
||||
run: bun test --timeout=60000 test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
run: bun test test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
|
||||
tier1:
|
||||
name: Tier 1 (Mechanical)
|
||||
@@ -91,7 +88,7 @@ jobs:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- name: Run Tier 1 E2E tests
|
||||
run: bun test --timeout=60000 test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
run: bun test test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
|
||||
@@ -158,7 +155,7 @@ jobs:
|
||||
}
|
||||
EOF
|
||||
- name: Run Tier 2 skill tests
|
||||
run: bun test --timeout=60000 test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
run: bun test test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
@@ -29,9 +29,7 @@ jobs:
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
# --timeout matches every scripts/ runner and covers hook budgets too
|
||||
# (bunfig.toml's timeout key is ignored by bun; hooks default to 5s).
|
||||
- run: bun test --timeout=60000
|
||||
- run: bun test
|
||||
- run: bun run verify
|
||||
- run: bun build --compile --target=${{ matrix.target }} --outfile bin/${{ matrix.artifact }} src/cli.ts
|
||||
- name: Attest build provenance
|
||||
|
||||
@@ -113,11 +113,6 @@ jobs:
|
||||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||||
- run: bun install
|
||||
- run: bun run verify
|
||||
# Guard: no bare `bun test` in workflows/scripts — bun ignores
|
||||
# bunfig.toml's timeout, and hooks (beforeAll/afterAll) get the 5s
|
||||
# default regardless of per-test third-arg timeouts. Runs directly
|
||||
# (not via verify's CHECKS array) to avoid a package.json edit.
|
||||
- run: bash scripts/check-bun-test-timeout.sh
|
||||
|
||||
serial-tests:
|
||||
# *.serial.test.ts at --max-concurrency=1. Lives in its own runner so
|
||||
|
||||
@@ -67,19 +67,6 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,690 +0,0 @@
|
||||
# Engine Dynamic-Import Reconciliation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Reconstruct the missing engine-path static-import hardening, preserve the four load-bearing lazy gateway fallbacks, and prevent unreviewed dynamic imports from returning.
|
||||
|
||||
**Architecture:** Make the 13 safe engine/migration import statements static and leave only four line-marked `ai/gateway.ts` imports inside their existing soft-failure `try/catch` boundaries. Enforce that current state with a repository-anchored Bash wrapper delegating to a fail-closed TypeScript AST scanner, a hermetic Bun regression test, package/verify wiring, and current-state architecture documentation.
|
||||
|
||||
**Tech Stack:** TypeScript compiler API, Bun test runner, Bash, Git, generated llms documentation bundles.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Reconstruct directly on branch `claude/kind-meitner-330c90`, based on investigated `origin/master` commit `6136e139972a5449630b4f47f5ed7b4cbe5b811b` plus design commit `d7f52d8c`.
|
||||
- Do not merge or cherry-pick `48ada48f`, `248bfe55`, `ef4cf7a8`, or either historical branch wholesale.
|
||||
- Do not modify `VERSION`, `CHANGELOG.md`, `TODOS.md`, or release metadata; this is a no-version-bump reconciliation.
|
||||
- Keep all four `await import('./ai/gateway.ts')` calls lazy: PGLite and Postgres `initSchema`, plus both `_upsertChunksOnce` methods.
|
||||
- Every allowed lazy gateway line must carry `engine-dynamic-import-ok`; there is no file-level exemption.
|
||||
- Preserve the stronger gateway rationale: the static closure is large, and eager module evaluation would occur outside the local `try/catch`, potentially converting a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
- Describe the hoists as engine-path hardening. Do not claim every dynamic import deterministically causes a Windows crash; system-wide commit exhaustion confounded prior measurements.
|
||||
- Keep shared PGLite/Postgres behavior in parity.
|
||||
- Invoke repository shell scripts through `bash` in `package.json`.
|
||||
- Capture complete test/check output to workspace-local `.context/*.txt` files before inspecting it; never pipe a test command directly through `head` or `tail`.
|
||||
- Use `git log -G`, not `git log -S`, for any additional dynamic-to-static import history work.
|
||||
- Keep every implementation and verification commit local. Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after local completion.
|
||||
- Before editing any affected function, run GBrain `code_blast` and `code_callers` for that symbol and inspect any disambiguation candidates.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
- Create `scripts/check-engine-dynamic-import.sh` — repository-anchored Bash wrapper for default and explicit input routing.
|
||||
- Create `scripts/check-engine-dynamic-import.ts` — TypeScript AST policy scanner for runtime `import()` expressions, parse/read failures, and exact-line comment-trivia opt-outs.
|
||||
- Create `test/scripts/check-engine-dynamic-import.test.ts` — 22 hermetic adversarial, CRLF, fail-closed, real-tree, and wiring tests.
|
||||
- Modify `src/core/pglite-engine.ts` — hoist three safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/postgres-engine.ts` — hoist eight safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/migrate.ts` — hoist two safe migration helper import statements.
|
||||
- Modify `package.json` — expose `check:engine-dynamic-import` and append it to `check:all` through `bash`.
|
||||
- Modify `scripts/run-verify-parallel.sh` — add the package check to the authoritative verify dispatcher.
|
||||
- Modify `CLAUDE.md` — add the cross-cutting current-state invariant.
|
||||
- Modify `docs/architecture/KEY_FILES.md` — update current-state entries for the three engine-path files.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` — required derived bundles after CLAUDE/reference documentation changes.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Establish and enforce the source invariant
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/check-engine-dynamic-import.sh`
|
||||
- Create: `scripts/check-engine-dynamic-import.ts`
|
||||
- Create: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `src/core/pglite-engine.ts`
|
||||
- Modify: `src/core/postgres-engine.ts`
|
||||
- Modify: `src/core/migrate.ts`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: shell positional arguments `FILE...`; without arguments, the guard scans the three repository files.
|
||||
- Produces: `scripts/check-engine-dynamic-import.sh [FILE...]`, exit `0` when every runtime dynamic import is allowed and exit `1` after reporting every `file:line:text` violation plus every read/parse error on stderr.
|
||||
- Produces: one line-level opt-out token, `engine-dynamic-import-ok`, accepted only in real comment trivia on the same physical line as the deliberately lazy import.
|
||||
- Fails closed on missing/unreadable inputs, TypeScript parse diagnostics, and scanner/process failures; comments, strings, templates, regex literals, and type-position `import(...)` syntax are not runtime imports.
|
||||
|
||||
- [ ] **Step 1: Record call-graph blast radius before touching functions**
|
||||
|
||||
First call `sources_list` and select the source whose registered path is this gbrain checkout. Then run `code_blast` and `code_callers` for these qualified symbols with that exact `source_id`, following `did_you_mean`/`candidates` when a method name is ambiguous:
|
||||
|
||||
```text
|
||||
src/core/pglite-engine.ts::PGLiteEngine.initSchema
|
||||
src/core/pglite-engine.ts::PGLiteEngine.batchRetry
|
||||
src/core/pglite-engine.ts::PGLiteEngine._upsertChunksOnce
|
||||
src/core/pglite-engine.ts::PGLiteEngine.mergeOntologyFact
|
||||
src/core/pglite-engine.ts::PGLiteEngine.getRecentSalience
|
||||
src/core/postgres-engine.ts::PostgresEngine.disconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.initSchema
|
||||
src/core/postgres-engine.ts::PostgresEngine.batchRetry
|
||||
src/core/postgres-engine.ts::PostgresEngine._upsertChunksOnce
|
||||
src/core/postgres-engine.ts::PostgresEngine.mergeOntologyFact
|
||||
src/core/postgres-engine.ts::PostgresEngine.reconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.getRecentSalience
|
||||
src/core/migrate.ts::runMigrationSQLWithRetry
|
||||
src/core/migrate.ts::runMigrations
|
||||
```
|
||||
|
||||
Use `depth: 5`, `max_nodes: 200`, and `limit: 100`. Expected: no caller requires a signature or behavior change; the patch only changes module binding time and retains all local fallback/error handling.
|
||||
|
||||
- [ ] **Step 2: Write the failing guard regression test**
|
||||
|
||||
Create `test/scripts/check-engine-dynamic-import.test.ts` as a hermetic subprocess suite. The completed 22-test surface covers:
|
||||
|
||||
- unmarked runtime `import()` rejection, including bare and trivia-separated forms;
|
||||
- same-line markers in real line or multiline block-comment trivia;
|
||||
- rejection of markers on prior lines or inside strings, templates, and module paths;
|
||||
- comments and comment-like delimiters inside strings, templates, and regex literals;
|
||||
- live code after same-line or multiline block comments close;
|
||||
- CRLF input and complete multi-file violation aggregation;
|
||||
- missing/readable mixed inputs and TypeScript parse diagnostics;
|
||||
- default repository anchoring when invoked from a foreign Git repository;
|
||||
- the reconciled three-file source scan plus package/parallel-verifier wiring.
|
||||
|
||||
Use the TypeScript parser rather than a partial lexical reimplementation. On Windows, set the test default to 30 seconds because each case launches Git Bash and Bun, whose startup can exceed Bun's 5-second per-test default.
|
||||
|
||||
- [ ] **Step 3: Run the test to prove the pre-implementation red state**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result captured inside the log. At minimum, the `exists` assertion fails because `scripts/check-engine-dynamic-import.sh` does not exist. Read `.context/engine-dynamic-import-red.txt`; do not infer the result from a truncated pipeline.
|
||||
|
||||
- [ ] **Step 4: Add the CRLF-safe, fail-closed guard**
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.sh` as a thin LF-terminated wrapper. Resolve its own directory first; when no explicit files are passed, anchor the repository with `git -C "$SCRIPT_DIR/.."` and scan the two engines plus `migrate.ts`. Delegate with `exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"` so scanner failures propagate.
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.ts` using the TypeScript compiler API:
|
||||
|
||||
- read every requested file and aggregate read failures;
|
||||
- parse as TypeScript and aggregate parse diagnostics;
|
||||
- walk the AST for `CallExpression`s whose expression is `ImportKeyword`;
|
||||
- locate all marker occurrences in the full source and use `ts.getTokenAtPosition` to admit only occurrences outside AST tokens (real comment trivia), recording their physical source lines;
|
||||
- require each runtime import's line to have an admitted marker or report its original `file:line:text`;
|
||||
- print every read/parse error and every violation before exiting nonzero.
|
||||
|
||||
This preserves CRLF line accounting, ignores comment/literal/type-only false positives, catches every legal runtime `import()` shape the TypeScript parser recognizes, rejects marker spoofing, and fails closed.
|
||||
|
||||
- [ ] **Step 5: Run the guard test to prove the source-tree midpoint is still red**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-midpoint.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: the synthetic violation, marker, comments, and CRLF cases pass. The default repository scan fails and reports all 17 current imports: 13 unmarked safe candidates plus the four not-yet-marked gateway calls.
|
||||
|
||||
- [ ] **Step 6: Hoist the three safe PGLite import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add the ontology/recency imports near the top of `src/core/pglite-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
```
|
||||
|
||||
Delete only these three in-method destructuring imports, leaving their uses unchanged:
|
||||
|
||||
```ts
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Mark both PGLite gateway soft-failure boundaries**
|
||||
|
||||
In `PGLiteEngine.initSchema`, preserve the `try/catch` and accessors, changing only the rationale and import line:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PGLiteEngine._upsertChunksOnce`, preserve the config-row and compile-time fallback chain:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Hoist the eight safe Postgres import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add these imports near the top of `src/core/postgres-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import { isConnectionEndedError } from './retry-matcher.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { logDbDisconnect } from './audit/db-disconnect-audit.ts';
|
||||
import { logPoolRecovery } from './audit/pool-recovery-audit.ts';
|
||||
```
|
||||
|
||||
Delete the eight safe dynamic-import statements while keeping their surrounding `try/catch` blocks and calls unchanged:
|
||||
|
||||
```ts
|
||||
const { logDbDisconnect } = await import('./audit/db-disconnect-audit.ts');
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { isConnectionEndedError } = await import('./retry-matcher.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
Update the stale `batchRetry` comment from “Lazy-import to avoid a circular dep concern” to current truth:
|
||||
|
||||
```ts
|
||||
// retry.ts is already in this module's static graph through withRetry, so
|
||||
// classifying the exhausted error does not need a second runtime import.
|
||||
```
|
||||
|
||||
- [ ] **Step 9: Mark both Postgres gateway soft-failure boundaries**
|
||||
|
||||
In `PostgresEngine.initSchema`, mirror the PGLite rationale and preserve behavior:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PostgresEngine._upsertChunksOnce`, preserve the DB-config fallback:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 10: Hoist the two migration helper import statements**
|
||||
|
||||
Add these static imports at the top of `src/core/migrate.ts`:
|
||||
|
||||
```ts
|
||||
// runMigrations executes while an initialized engine is live. Keep its helper
|
||||
// modules in the static graph rather than importing them from async handlers.
|
||||
import {
|
||||
isStatementTimeoutError,
|
||||
isRetryableConnError,
|
||||
} from './retry-matcher.ts';
|
||||
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
|
||||
```
|
||||
|
||||
Delete only these two local destructuring imports:
|
||||
|
||||
```ts
|
||||
const { isStatementTimeoutError, isRetryableConnError } = await import('./retry-matcher.ts');
|
||||
const { repairTimelineDedupIndex } = await import('./timeline-dedup-repair.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 11: Run the complete guard test and direct guard**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; output contains `check-engine-dynamic-import: ok (3 file(s) scanned)`.
|
||||
|
||||
- [ ] **Step 12: Prove the guard leaves exactly four marked dynamic imports**
|
||||
|
||||
```bash
|
||||
git grep -n -F "import('./ai/gateway.ts'); // engine-dynamic-import-ok" -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts > .context/engine-dynamic-import-sites.txt; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exactly four lines, all importing `./ai/gateway.ts` and all carrying `engine-dynamic-import-ok`; no match in `src/core/migrate.ts`.
|
||||
|
||||
- [ ] **Step 13: Run focused behavior tests**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. If Windows resource pressure aborts the process, record the exact exit code and rerun the failing file alone; do not relabel an infrastructure abort as a source pass.
|
||||
|
||||
- [ ] **Step 14: Commit the source invariant locally**
|
||||
|
||||
```bash
|
||||
git add scripts/check-engine-dynamic-import.sh scripts/check-engine-dynamic-import.ts test/scripts/check-engine-dynamic-import.test.ts src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "fix(engine): reconcile dynamic import hardening"
|
||||
```
|
||||
|
||||
Expected: one local commit; no version or release files staged.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Wire the guard into repository checks
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `package.json`
|
||||
- Modify: `scripts/run-verify-parallel.sh`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `scripts/check-engine-dynamic-import.sh` from Task 1.
|
||||
- Produces: package script `check:engine-dynamic-import` and verify dry-list entry of the same name.
|
||||
|
||||
- [ ] **Step 1: Add failing wiring assertions**
|
||||
|
||||
Add these imports/constants to `test/scripts/check-engine-dynamic-import.test.ts`:
|
||||
|
||||
```ts
|
||||
const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json');
|
||||
```
|
||||
|
||||
Append this test block:
|
||||
|
||||
```ts
|
||||
describe('engine dynamic-import guard wiring', () => {
|
||||
it('is invoked through bash by check:all', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf8')) as {
|
||||
scripts: Record<string, string>;
|
||||
};
|
||||
expect(pkg.scripts['check:engine-dynamic-import']).toBe(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
expect(pkg.scripts['check:all']).toContain(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
});
|
||||
|
||||
it('is listed by the authoritative verify dispatcher', () => {
|
||||
const result = spawnSync(BASH, [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(new Set((result.stdout ?? '').trim().split('\n'))).toContain(
|
||||
'check:engine-dynamic-import',
|
||||
);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the test and verify both wiring assertions fail**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result. The source guard tests remain green; package-script and verify-list assertions fail because the wiring is absent.
|
||||
|
||||
- [ ] **Step 3: Add the package scripts**
|
||||
|
||||
In `package.json`, add this script alongside the other `check:*` entries:
|
||||
|
||||
```json
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh"
|
||||
```
|
||||
|
||||
Append the guard to the existing `check:all` chain, preserving every existing check:
|
||||
|
||||
```text
|
||||
&& bash scripts/check-engine-dynamic-import.sh
|
||||
```
|
||||
|
||||
Do not rewrite any existing shell entry without its `bash` prefix.
|
||||
|
||||
- [ ] **Step 4: Add the authoritative verify entry**
|
||||
|
||||
In `scripts/run-verify-parallel.sh`, add this stable `CHECKS` entry near the other source-shape guards:
|
||||
|
||||
```bash
|
||||
"check:engine-dynamic-import"
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run the regression test and package check**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bun run check:engine-dynamic-import > .context/engine-dynamic-import-package-check.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0` and three files scanned.
|
||||
|
||||
- [ ] **Step 6: Commit the wiring locally**
|
||||
|
||||
```bash
|
||||
git add package.json scripts/run-verify-parallel.sh test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "test(engine): guard dynamic import policy"
|
||||
```
|
||||
|
||||
Expected: one local commit with the guard wiring and its regression assertions.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Document the current-state invariant
|
||||
|
||||
**Files:**
|
||||
- Modify: `CLAUDE.md`
|
||||
- Modify: `docs/architecture/KEY_FILES.md`
|
||||
- Regenerate: `llms.txt`
|
||||
- Regenerate: `llms-full.txt`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the four-marked-import source state and the `check:engine-dynamic-import` package surface.
|
||||
- Produces: current-state contributor guidance and fresh generated documentation bundles.
|
||||
|
||||
- [ ] **Step 1: Add the cross-cutting invariant to `CLAUDE.md`**
|
||||
|
||||
Add this bullet under “Cross-cutting invariants” near the other language/filesystem guards:
|
||||
|
||||
```md
|
||||
- **Engine-live paths use static imports by default.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, helper modules are top-level imports. The only current
|
||||
exceptions are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
```
|
||||
|
||||
Do not add release tags, Windows-crash certainty, or historical branch names.
|
||||
|
||||
- [ ] **Step 2: Update the PGLite current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this current-state sentence to the existing `src/core/pglite-engine.ts` entry, preserving the entry as one bullet:
|
||||
|
||||
```md
|
||||
Engine-path helper dependencies (`retry`, ontology, recency decay) bind statically; the only lazy imports are `ai/gateway.ts` in `initSchema` and `_upsertChunksOnce`, line-marked because their local catches preserve compiled-default and stored-config fallbacks that eager module evaluation would bypass.
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update the Postgres current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the existing `src/core/postgres-engine.ts` entry:
|
||||
|
||||
```md
|
||||
Retry classifiers, ontology/recency helpers, and disconnect/pool-recovery audit writers bind statically; only the two `ai/gateway.ts` fallback lookups stay lazy and line-marked, in parity with PGLite.
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update the migration current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the canonical `src/core/migrate.ts` entry (the broad runner entry, not the older v95-specific index note):
|
||||
|
||||
```md
|
||||
`retry-matcher.ts` and `timeline-dedup-repair.ts` are static dependencies because `runMigrations()` executes from live engine initialization; the engine dynamic-import guard scans this file with both engine implementations.
|
||||
```
|
||||
|
||||
Keep all three entries current-state only: no `v0.42.x`, branch, commit, “previously,” or “was/now” narration.
|
||||
|
||||
- [ ] **Step 5: Regenerate the llms bundles**
|
||||
|
||||
```bash
|
||||
bun run build:llms > .context/engine-dynamic-import-build-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; `llms.txt` and/or `llms-full.txt` update according to their configured linked/inlined status. Byte-identical output for a linked source is acceptable; the freshness test is authoritative.
|
||||
|
||||
- [ ] **Step 6: Run documentation freshness checks**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-llms-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
```bash
|
||||
bun run check:doc-history > .context/engine-dynamic-import-doc-history.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; no release-history marker is introduced into current-state reference docs.
|
||||
|
||||
- [ ] **Step 7: Confirm prohibited release files remain untouched**
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD -- VERSION CHANGELOG.md TODOS.md
|
||||
```
|
||||
|
||||
Expected: no output.
|
||||
|
||||
- [ ] **Step 8: Commit documentation and generated bundles locally**
|
||||
|
||||
```bash
|
||||
git add CLAUDE.md docs/architecture/KEY_FILES.md llms.txt llms-full.txt
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs(engine): record static import invariant"
|
||||
```
|
||||
|
||||
Expected: one local documentation commit. If one generated bundle is byte-identical, Git simply omits it.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Verify and review the complete local reconciliation
|
||||
|
||||
**Files:**
|
||||
- Verify all files changed since `d7f52d8c`.
|
||||
- Do not create or modify release/publication metadata.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Tasks 1–3.
|
||||
- Produces: full local verification evidence and an implementation diff ready for user review, not publication.
|
||||
|
||||
- [ ] **Step 1: Run the regression test and direct guard again**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-final-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-final-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; three files scanned.
|
||||
|
||||
- [ ] **Step 2: Run TypeScript checking**
|
||||
|
||||
```bash
|
||||
bun run typecheck > .context/engine-dynamic-import-typecheck.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. Report exact diagnostics if the branch or current Windows environment has a pre-existing failure.
|
||||
|
||||
- [ ] **Step 3: Run the authoritative verify dispatcher**
|
||||
|
||||
```bash
|
||||
bun run verify > .context/engine-dynamic-import-verify.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`, including `check:engine-dynamic-import`. On Windows, classify any per-check timeout from the complete log instead of treating the aggregate result as a source regression without evidence.
|
||||
|
||||
- [ ] **Step 4: Re-run focused tests as an ownership check**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-final-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; record any infrastructure abort separately and rerun only the named file before classifying it.
|
||||
|
||||
- [ ] **Step 5: Run the llms freshness test after all documentation settles**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-final-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
- [ ] **Step 6: Run whitespace and scope checks**
|
||||
|
||||
```bash
|
||||
git diff --check d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected: exit `0`, no output.
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected files only:
|
||||
|
||||
```text
|
||||
CLAUDE.md
|
||||
docs/architecture/KEY_FILES.md
|
||||
docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
llms-full.txt
|
||||
llms.txt
|
||||
package.json
|
||||
scripts/check-engine-dynamic-import.sh
|
||||
scripts/check-engine-dynamic-import.ts
|
||||
scripts/run-verify-parallel.sh
|
||||
src/core/migrate.ts
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
Either generated llms file may be absent if regeneration proves it byte-identical. `VERSION`, `CHANGELOG.md`, and `TODOS.md` must be absent.
|
||||
|
||||
- [ ] **Step 7: Review the exact implementation diff**
|
||||
|
||||
```bash
|
||||
git diff --stat d7f52d8c..HEAD && git diff d7f52d8c..HEAD -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts scripts/check-engine-dynamic-import.sh test/scripts/check-engine-dynamic-import.test.ts package.json scripts/run-verify-parallel.sh CLAUDE.md docs/architecture/KEY_FILES.md
|
||||
```
|
||||
|
||||
Expected review findings:
|
||||
|
||||
- Exactly 13 safe `await import(...)` statements are removed.
|
||||
- Exactly four `ai/gateway.ts` imports remain, all marked on the same line.
|
||||
- All four gateway imports remain inside their original local `try/catch` fallback boundaries.
|
||||
- No accessor logic, fallback ordering, SQL, public signature, or engine parity behavior changes.
|
||||
- The parser-backed guard reports all violations plus read/parse failures, preserves CRLF line accounting, ignores comments/literals/type-only syntax, detects every runtime `import()` call expression, and accepts opt-outs only from real comment trivia on the same physical line.
|
||||
- The package script invokes the shell guard through Bash; `check:all` invokes that shell guard directly, and the parallel verify dispatcher invokes the package check.
|
||||
- Documentation is current-state and makes no deterministic Windows-crash claim.
|
||||
|
||||
**Observed Windows verification classification:** The authoritative aggregate completed with 25 of 33 checks passing. Individual reruns showed `check:test-names` and `typecheck` green; privacy/isolation exceeded Windows timing budgets; WASM failed in unrelated temporary-symlink setup; eval-glossary was CRLF/LF drift; resolver/brain-first findings predated and did not intersect this branch. The focused aggregate produced 103 pass / 5 fail: three setup-hook timeouts reproduced at the untouched base, and the known `migrate-retry` polling failure reproduced there. Its additional race-status assertion did not reproduce at base, so it remains an unresolved timing-sensitive limitation in untouched code—not evidence of an in-scope defect and not claimed as conclusively pre-existing.
|
||||
|
||||
- [ ] **Step 8: Commit the approved plan document locally**
|
||||
|
||||
The plan is an approved, tracked execution artifact and must not be left as an uncommitted file after implementation:
|
||||
|
||||
```bash
|
||||
git add docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs: plan engine dynamic-import reconciliation"
|
||||
```
|
||||
|
||||
Expected: one local plan commit; no release metadata staged.
|
||||
|
||||
- [ ] **Step 9: Inspect final status without publishing**
|
||||
|
||||
```bash
|
||||
git status --short --branch
|
||||
```
|
||||
|
||||
Expected: branch `claude/kind-meitner-330c90` with a clean working tree. No push, PR, upstream comment, or other external side effect.
|
||||
|
||||
- [ ] **Step 10: Capture the completed milestone to memory**
|
||||
|
||||
Before writing, search MemPalace wing `gbrain` for this exact reconciliation to avoid duplication. Add a verbatim drawer recording exact base/head commits, the 13 hoists, four gateway opt-outs and rationale, guard/test/docs files, every verification command with exit code, and any environment-owned failures. Add a GBrain project timeline entry only if there is an existing relevant gbrain project page; do not create duplicate release metadata.
|
||||
|
||||
- [ ] **Step 11: Report the local result and ask separately before publication**
|
||||
|
||||
Report:
|
||||
|
||||
- exact local commits;
|
||||
- changed files;
|
||||
- test/check exit codes;
|
||||
- any blocked or pre-existing failures;
|
||||
- confirmation that release files were untouched;
|
||||
- confirmation that nothing was pushed or published.
|
||||
|
||||
Do not run any publication command. Wait for explicit user approval before any push, PR, or upstream interaction.
|
||||
@@ -1,142 +0,0 @@
|
||||
# Engine dynamic-import reconciliation design
|
||||
|
||||
**Date:** 2026-07-28
|
||||
|
||||
## Goal
|
||||
|
||||
Reconcile the overlapping engine dynamic-import changes from:
|
||||
|
||||
- `claude/hungry-edison-8bb1cd` at release commits `48ada48f` and `248bfe55`
|
||||
- `claude/elegant-gates-e5275e` at `ef4cf7a8`
|
||||
|
||||
onto a fresh branch from current `origin/master`, without merging or cherry-picking either lineage wholesale and without adding a release/version bump.
|
||||
|
||||
## Established state
|
||||
|
||||
At investigation time:
|
||||
|
||||
- `origin/master` was `6136e139972a5449630b4f47f5ed7b4cbe5b811b`, version `0.42.67.0`.
|
||||
- Upstream PR #3511 was still open, so trunk did not contain its two `chronicle/ontology.ts` hoists.
|
||||
- Neither source branch was an ancestor of trunk.
|
||||
- Trunk contained 17 dynamic imports in the three engine-path files:
|
||||
- 13 safe-hoist candidates: two ontology imports, nine engine helper/audit imports, and two migration imports.
|
||||
- Four `ai/gateway.ts` imports, all inside `try/catch` fallback paths.
|
||||
- `git log -G` showed the separate ontology, helper, migration, and gateway histories. `git log -S` is not suitable for this dynamic-to-static replacement because the relevant token can remain present while its context changes.
|
||||
- The guard from `ef4cf7a8` passed against that commit but failed against trunk. It also knew about only two gateway opt-outs because two `_upsertChunksOnce` gateway lookups landed later in trunk.
|
||||
|
||||
## Selected approach
|
||||
|
||||
Reconstruct the intended current state directly on fresh `origin/master`.
|
||||
|
||||
Do not merge or cherry-pick either old lineage. Selectively reproduce the desired source changes, adapt the guard to the current four gateway call sites, and write current-state documentation. This avoids importing stale release metadata, stale TODO claims, and unrelated lineage changes.
|
||||
|
||||
## Source changes
|
||||
|
||||
### Safe static imports
|
||||
|
||||
Hoist all 13 safe candidates:
|
||||
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `valueHash`, `normalizeDimension`, `isNovelDimension` from `chronicle/ontology.ts`
|
||||
- `isRetryableConnError` through the existing `retry.ts` import
|
||||
- `resolveRecencyDecayMap`, `DEFAULT_FALLBACK` from `search/recency-decay.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- the same ontology, retry, and recency helpers
|
||||
- `isConnectionEndedError` from `retry-matcher.ts`
|
||||
- `logDbDisconnect` from `audit/db-disconnect-audit.ts`
|
||||
- `logPoolRecovery` from `audit/pool-recovery-audit.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- `isStatementTimeoutError`, `isRetryableConnError` from `retry-matcher.ts`
|
||||
- `repairTimelineDedupIndex` from `timeline-dedup-repair.ts`
|
||||
|
||||
The implementation must keep the two engines in parity where the behavior is shared. Comments should describe current invariants, not repeat an unproven causal claim that these hoists fix the Windows test-runner crash.
|
||||
|
||||
### Deliberately lazy gateway imports
|
||||
|
||||
Keep all four `await import('./ai/gateway.ts')` call sites lazy:
|
||||
|
||||
- PGLite `initSchema`
|
||||
- PGLite `_upsertChunksOnce`
|
||||
- Postgres `initSchema`
|
||||
- Postgres `_upsertChunksOnce`
|
||||
|
||||
Each line receives the explicit `engine-dynamic-import-ok` marker and a concise nearby rationale.
|
||||
|
||||
The rationale has two parts:
|
||||
|
||||
1. The gateway's static closure includes the AI SDK, provider packages, and validation/config machinery, so eager loading would tax engine startup paths that do not otherwise need it.
|
||||
2. More importantly, each lookup is inside a `try/catch` that preserves a soft fallback (compiled defaults or the brain's stored embedding-model config). Hoisting the module would evaluate it before that catch can run and could convert a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
|
||||
The guard must not allow unmarked gateway imports or a broad file-level exemption.
|
||||
|
||||
## Guard and wiring
|
||||
|
||||
Add `scripts/check-engine-dynamic-import.sh`, adapted from `ef4cf7a8`, with these properties:
|
||||
|
||||
- Default scan set:
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- Normalize trailing CR before matching so CRLF checkouts cannot bypass the check.
|
||||
- Ignore comment-only lines.
|
||||
- Ignore only lines carrying `engine-dynamic-import-ok`.
|
||||
- Report every unmarked `await import(` with file and line.
|
||||
- Explain that contributors should prefer a static import and must justify a real opt-out.
|
||||
- Avoid asserting that every dynamic import deterministically crashes Windows; the measured evidence supports treating the pattern as an engine-path hardening invariant, while box-level commit exhaustion remained a confound in prior runs.
|
||||
|
||||
Wire it into:
|
||||
|
||||
- `package.json` as `check:engine-dynamic-import`
|
||||
- `package.json` `check:all`
|
||||
- `scripts/run-verify-parallel.sh`
|
||||
|
||||
Follow trunk's current rule that package scripts invoke repository shell scripts through `bash`.
|
||||
|
||||
## Regression coverage
|
||||
|
||||
Add an automated test for the guard. It must cover:
|
||||
|
||||
- A real dynamic import produces exit 1 and is reported.
|
||||
- A line carrying `engine-dynamic-import-ok` is allowed.
|
||||
- Line comments and block-comment lines do not produce findings.
|
||||
- The same violation is caught with CRLF input.
|
||||
- The default repository scan passes after the source reconciliation.
|
||||
|
||||
Use a temporary fixture rather than mutating tracked source files. Keep assertions path-portable.
|
||||
|
||||
The pre-fix red demonstration is the exact guard from `ef4cf7a8` run against current trunk: it exits 1 and reports the existing unmarked imports. The post-fix guard and test must pass.
|
||||
|
||||
## Documentation policy
|
||||
|
||||
Preserve current behavior, not either old release narrative:
|
||||
|
||||
- Do not modify `VERSION` or add a release `CHANGELOG.md` entry.
|
||||
- Do not copy old version headings or completed release TODO blocks.
|
||||
- Do not retain the old TODO claiming that extracting gateway accessors is necessarily the fix; the lazy imports are deliberately protected by their local soft-failure boundaries.
|
||||
- Add the cross-cutting no-unmarked-dynamic-import invariant to `CLAUDE.md`.
|
||||
- Update the current-state entries for `src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and `src/core/migrate.ts` in `docs/architecture/KEY_FILES.md` where needed.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` after the documentation edits.
|
||||
- Add a TODO only if implementation uncovers a real unresolved action.
|
||||
|
||||
Public documentation must use generic language and must not overstate the historical Windows crash causality.
|
||||
|
||||
## Verification
|
||||
|
||||
Capture full output to files before inspecting summaries. Run, at minimum:
|
||||
|
||||
1. The guard regression test.
|
||||
2. `bash scripts/check-engine-dynamic-import.sh`.
|
||||
3. Focused tests that exercise the touched engine, migration, retry, audit, and recency modules.
|
||||
4. `bun run typecheck`.
|
||||
5. `bun run verify`.
|
||||
6. `bun run build:llms` followed by `bun test test/build-llms.test.ts`.
|
||||
7. `git diff --check` and a final clean-status/diff review.
|
||||
|
||||
If platform contention or existing Windows suite defects block a broad test, report the exact command, exit code, and ownership classification rather than declaring success from a partial run.
|
||||
|
||||
## Git and publication boundary
|
||||
|
||||
- Work on `claude/kind-meitner-330c90`, reset locally to the exact investigated `origin/master` base.
|
||||
- Preserve the previous worktree tip under `claude/kind-meitner-330c90-pre-reconcile`.
|
||||
- Keep implementation and verification commits local.
|
||||
- Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after the local result is complete.
|
||||
@@ -216,19 +216,6 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
|
||||
+1
-2
@@ -48,8 +48,7 @@
|
||||
"check:system-of-record": "bash scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "bash scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "bash scripts/check-cli-executable.sh",
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:all": "bash scripts/check-privacy.sh && bash scripts/check-proposal-pii.sh && bash scripts/check-test-real-names.sh && bash scripts/check-jsonb-pattern.sh && bash scripts/check-source-id-projection.sh && bash scripts/check-source-config-leak.sh && bash scripts/check-progress-to-stdout.sh && bash scripts/check-no-tracked-symlinks.sh && bash scripts/check-no-legacy-getconnection.sh && bash scripts/check-test-isolation.sh && bash scripts/check-trailing-newline.sh && bash scripts/check-wasm-embedded.sh && bash scripts/check-exports-count.sh && bash scripts/check-admin-build.sh && bash scripts/check-admin-scope-drift.sh && bash scripts/check-cli-executable.sh && bash scripts/check-skill-brain-first.sh && bash scripts/check-operations-filter-bypass.sh && bash scripts/check-gateway-routed-no-direct-anthropic.sh && bash scripts/check-worker-pool-atomicity.sh && bash scripts/check-key-files-current-state.sh && bash scripts/check-no-double-retry.sh && bash scripts/check-batch-audit-site.sh && bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:all": "bash scripts/check-privacy.sh && bash scripts/check-proposal-pii.sh && bash scripts/check-test-real-names.sh && bash scripts/check-jsonb-pattern.sh && bash scripts/check-source-id-projection.sh && bash scripts/check-source-config-leak.sh && bash scripts/check-progress-to-stdout.sh && bash scripts/check-no-tracked-symlinks.sh && bash scripts/check-no-legacy-getconnection.sh && bash scripts/check-test-isolation.sh && bash scripts/check-trailing-newline.sh && bash scripts/check-wasm-embedded.sh && bash scripts/check-exports-count.sh && bash scripts/check-admin-build.sh && bash scripts/check-admin-scope-drift.sh && bash scripts/check-cli-executable.sh && bash scripts/check-skill-brain-first.sh && bash scripts/check-operations-filter-bypass.sh && bash scripts/check-gateway-routed-no-direct-anthropic.sh && bash scripts/check-worker-pool-atomicity.sh && bash scripts/check-key-files-current-state.sh && bash scripts/check-no-double-retry.sh && bash scripts/check-batch-audit-site.sh",
|
||||
"check:gateway-routed": "bash scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "bash scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "bash scripts/check-key-files-current-state.sh",
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: every `bun test` invocation in workflows and runner scripts must
|
||||
# pass an explicit --timeout.
|
||||
#
|
||||
# Why: bun ignores bunfig.toml's `timeout` key (verified on 1.3.14), so a bare
|
||||
# `bun test` gets the 5000ms default for BOTH tests and beforeAll/beforeEach/
|
||||
# afterAll/afterEach hooks. Hooks do NOT inherit a test's third-arg timeout —
|
||||
# a file whose tests all declare `}, 30_000)` still has a 5s hook budget, and
|
||||
# slow setup (Postgres connect + migrations, PGLite cold start) flakes on
|
||||
# loaded CI runners with the signature `(unnamed) [5001ms] ... hook timed out`
|
||||
# (the #3545 jsonb-parity failure). The CLI --timeout flag is the one measured
|
||||
# mechanism that raises the hook budget uniformly; per-hook second-arg
|
||||
# timeouts work too but don't scale to ~400 slow hooks.
|
||||
#
|
||||
# Usage: scripts/check-bun-test-timeout.sh
|
||||
# Exit: 0 when clean, 1 when a bare `bun test` invocation is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Match executable `bun test` invocations. Exclude comment lines (#, //, *)
|
||||
# and lines that already carry --timeout anywhere.
|
||||
# Scope: workflows + runner scripts (the surfaces CI executes). package.json
|
||||
# script bodies route through scripts/ already; editing it is out of scope here.
|
||||
violations="$(grep -rnE '\bbun test\b' .github/workflows scripts 2>/dev/null \
|
||||
| grep -v -- '--timeout' \
|
||||
| grep -vE ':[[:space:]]*(#|//|\*)' \
|
||||
| grep -v 'check-bun-test-timeout' \
|
||||
|| true)"
|
||||
|
||||
if [ -n "$violations" ]; then
|
||||
echo "FAIL: bare 'bun test' without --timeout (5s default kills slow setup hooks):" >&2
|
||||
echo "$violations" >&2
|
||||
echo "" >&2
|
||||
echo "Add --timeout=60000 (see scripts/run-unit-shard.sh for the convention)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: every bun test invocation passes an explicit --timeout."
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Engine-live paths use static imports by default. A line-level
|
||||
# `engine-dynamic-import-ok` marker is required for a justified lazy import.
|
||||
#
|
||||
# Historical Windows runs associated imports on these paths with abrupt Bun
|
||||
# test-process exits, but system-wide commit exhaustion remained a confound.
|
||||
# This guard therefore enforces a reviewed engine-path hardening invariant; it
|
||||
# does not claim every dynamic import deterministically crashes Windows.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/check-engine-dynamic-import.sh
|
||||
# bash scripts/check-engine-dynamic-import.sh FILE [FILE...]
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" || exit 1
|
||||
|
||||
if [ "$#" -gt 0 ]; then
|
||||
FILES=("$@")
|
||||
else
|
||||
ROOT="$(git -C "$SCRIPT_DIR/.." rev-parse --show-toplevel 2>/dev/null || true)"
|
||||
[ -n "$ROOT" ] || ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$ROOT" || exit 1
|
||||
FILES=(
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
src/core/migrate.ts
|
||||
)
|
||||
fi
|
||||
|
||||
exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"
|
||||
@@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import ts from 'typescript';
|
||||
|
||||
const MARKER = 'engine-dynamic-import-ok';
|
||||
const MARKER_TOKEN_CHAR = /[\p{ID_Continue}$-]/u;
|
||||
const files = process.argv.slice(2);
|
||||
const violations: string[] = [];
|
||||
const readErrors: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
let sourceText: string;
|
||||
try {
|
||||
sourceText = await readFile(file, 'utf8');
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
readErrors.push(`ERROR: cannot read input file ${file}: ${detail}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceFile = ts.createSourceFile(
|
||||
file,
|
||||
sourceText,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
const lines = sourceText.split(/\r?\n/);
|
||||
const markerLines = new Set<number>();
|
||||
|
||||
if (sourceFile.parseDiagnostics.length > 0) {
|
||||
const diagnostics = sourceFile.parseDiagnostics
|
||||
.map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, ' '))
|
||||
.join('; ');
|
||||
readErrors.push(`ERROR: cannot parse input file ${file}: ${diagnostics}`);
|
||||
}
|
||||
|
||||
for (let markerPos = sourceText.indexOf(MARKER); markerPos >= 0; markerPos = sourceText.indexOf(MARKER, markerPos + MARKER.length)) {
|
||||
const before = Array.from(sourceText.slice(0, markerPos)).at(-1);
|
||||
const after = Array.from(sourceText.slice(markerPos + MARKER.length))[0];
|
||||
const standaloneMarker = (!before || !MARKER_TOKEN_CHAR.test(before))
|
||||
&& (!after || !MARKER_TOKEN_CHAR.test(after));
|
||||
const token = ts.getTokenAtPosition(sourceFile, markerPos);
|
||||
const insideToken = token.getStart(sourceFile) <= markerPos && markerPos < token.end;
|
||||
if (standaloneMarker && !insideToken) {
|
||||
markerLines.add(sourceFile.getLineAndCharacterOfPosition(markerPos).line);
|
||||
}
|
||||
}
|
||||
|
||||
function visit(node: ts.Node): void {
|
||||
if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) {
|
||||
const { line } = sourceFile.getLineAndCharacterOfPosition(node.expression.getStart(sourceFile));
|
||||
const sourceLine = lines[line] ?? '';
|
||||
if (!markerLines.has(line)) {
|
||||
violations.push(` ${file}:${line + 1}:${sourceLine}`);
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
|
||||
visit(sourceFile);
|
||||
}
|
||||
|
||||
for (const error of readErrors) console.error(error);
|
||||
|
||||
if (violations.length > 0) {
|
||||
console.error('ERROR: unreviewed dynamic import on an engine-live path:');
|
||||
console.error();
|
||||
console.error(violations.join('\n'));
|
||||
console.error();
|
||||
console.error('Prefer a static top-level import. If lazy loading is load-bearing,');
|
||||
console.error("append 'engine-dynamic-import-ok' to that exact line and document");
|
||||
console.error('the startup or soft-failure boundary that requires it.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (readErrors.length > 0) process.exit(1);
|
||||
|
||||
console.log(`check-engine-dynamic-import: ok (${files.length} file(s) scanned)`);
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Import an envelope-v0 file (a JSON serialization of AI chat history; format
|
||||
* spec: github.com/memvelope/memvelope) into a brain repo as one Markdown page
|
||||
* per conversation, which `gbrain sync` ingests.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/envelope-to-gbrain.mjs <envelope.mve.json> [outDir]
|
||||
*
|
||||
* Zero dependencies. Deterministic. No network. It does NOT call gbrain — it
|
||||
* only writes Markdown files.
|
||||
*
|
||||
* Output layout:
|
||||
* - One page per conversation, filename = date + conversation id (shared
|
||||
* titles cannot collide; the id is the natural key). A duplicate id
|
||||
* overwrites its own filename and warns on stderr; stdout reports DISTINCT
|
||||
* files written, not write calls.
|
||||
* - Frontmatter: `type: conversation` (keeps pages eligible for
|
||||
* conversation-facts extraction and chronicle behavior after sync), the
|
||||
* source provider, the conversation id, and `origin: memvelope/envelope-v0`.
|
||||
* - Page `date` is the first 10 chars of the conversation's ISO-8601
|
||||
* `created_at`. Body keeps message-id citations beside each speaker turn.
|
||||
*
|
||||
* Memory: the whole envelope is held in memory (no streaming); envelopes are
|
||||
* far smaller than the vendor exports they serialize.
|
||||
*
|
||||
* Verify:
|
||||
* node scripts/envelope-to-gbrain.mjs test/fixtures/memvelope/sample.mve.json /tmp/out
|
||||
* -> expect "wrote 1 markdown page(s)"
|
||||
* bun test test/envelope-to-gbrain.test.ts
|
||||
*
|
||||
* STATUS: live-verified against gbrain v0.42.56.0 on 2026-07-03: the sample
|
||||
* fixture -> 1 page; a real 662MB Claude export -> 353 conversations = 353
|
||||
* distinct pages (no collisions), searchable after sync with provenance and
|
||||
* message-id citations intact.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const [, , envelopePath, outDir = './brain/conversations'] = process.argv;
|
||||
if (!envelopePath) {
|
||||
console.error('usage: node envelope-to-gbrain.mjs <envelope.mve.json> [outDir]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const env = JSON.parse(readFileSync(envelopePath, 'utf8'));
|
||||
if (env.memvelope !== 'envelope-v0') {
|
||||
console.error(`not an envelope-v0 file (memvelope field = ${JSON.stringify(env.memvelope)})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const slug = (s, fallback) =>
|
||||
(String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback).slice(0, 60);
|
||||
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
const filesWritten = new Set();
|
||||
let collisions = 0;
|
||||
const conversations = env.conversations || [];
|
||||
for (const [i, c] of conversations.entries()) {
|
||||
const date = (c.created_at || '').slice(0, 10);
|
||||
// Name the file by the conversation's own id — the natural unique key — so two
|
||||
// conversations that share a date and title can never silently overwrite each
|
||||
// other. The date only leads as a human/chronological sort prefix; the id
|
||||
// carries uniqueness. Positional fallback keeps names unique and deterministic
|
||||
// when an envelope omits an id.
|
||||
// One predicate for "this conversation carries its own id", shared by the
|
||||
// filename and the frontmatter below. Keeping it in a single place is what
|
||||
// stops the two from disagreeing about whether an id exists.
|
||||
const hasId = typeof c.id === 'string' && c.id.trim() !== '';
|
||||
const convId = hasId ? c.id.trim() : `conv-${i + 1}`;
|
||||
const name = `${date || '0000-00-00'}-${slug(convId, `conv-${i + 1}`)}.md`;
|
||||
// gbrain reads YAML frontmatter + markdown body; keep provenance in frontmatter.
|
||||
// Emit `type: conversation` so gbrain stores these as conversation pages rather
|
||||
// than defaulting to the generic `concept`. gbrain is open-typed — it takes an
|
||||
// explicit frontmatter `type` verbatim — and its conversation-aware features
|
||||
// (conversation-facts extraction, the conversation_format_coverage check,
|
||||
// chronicle eligibility) key off `type == 'conversation'`.
|
||||
const front = [
|
||||
'---',
|
||||
'type: conversation',
|
||||
`title: ${JSON.stringify(c.title || 'Untitled conversation')}`,
|
||||
`date: ${date || 'null'}`,
|
||||
// Every interpolated value is quoted. An envelope is a third-party file, so
|
||||
// a provider string carrying a newline would otherwise close this scalar and
|
||||
// inject arbitrary frontmatter keys into the page gbrain ingests.
|
||||
`source: ${JSON.stringify(env.meta?.source_provider || 'unknown')}`,
|
||||
// Omit the key entirely when the envelope carries no id, rather than
|
||||
// emitting the literal `undefined` or a synthesized `conv-N` — the positional
|
||||
// fallback names the file, but it is not a memvelope conversation id and
|
||||
// must not be recorded as one.
|
||||
...(hasId ? [`memvelope_conversation_id: ${JSON.stringify(convId)}`] : []),
|
||||
'origin: memvelope/envelope-v0',
|
||||
'---',
|
||||
'',
|
||||
].join('\n');
|
||||
const body = (c.messages || [])
|
||||
.map((m) => `**${m.role === 'user' ? 'Me' : 'Assistant'}** (${m.ts || 'no timestamp'} · ${m.id}):\n\n${m.text}`)
|
||||
.join('\n\n---\n\n');
|
||||
// Never lose a page silently: if two conversations still map to the same
|
||||
// filename (e.g. an envelope carrying duplicate ids), warn loudly instead of
|
||||
// overwriting in silence, and report the count of DISTINCT files written — not
|
||||
// the number of write calls, which is what hid the old title-collision bug.
|
||||
if (filesWritten.has(name)) {
|
||||
collisions += 1;
|
||||
console.warn(`warning: filename collision on "${name}" — conversation id ${JSON.stringify(c.id)} is not unique; overwriting the earlier page.`);
|
||||
}
|
||||
writeFileSync(join(outDir, name), front + `# ${c.title || 'Conversation'}\n\n` + body + '\n');
|
||||
filesWritten.add(name);
|
||||
}
|
||||
console.log(`wrote ${filesWritten.size} markdown page(s) to ${outDir} — point gbrain's sync at this directory.`);
|
||||
if (collisions) {
|
||||
console.warn(`warning: ${collisions} filename collision(s) — ${collisions} page(s) overwritten. Deduplicate conversation ids in the envelope to avoid data loss.`);
|
||||
}
|
||||
+2
-3
@@ -162,9 +162,8 @@ for f in "${files[@]}"; do
|
||||
if [ -n "${DATABASE_URL:-}" ]; then
|
||||
psql "$DATABASE_URL" -At -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid != pg_backend_pid() AND datname = current_database()" >/dev/null 2>&1 || true
|
||||
fi
|
||||
# Hard outer timeout (180s per file). bun's --timeout covers tests AND
|
||||
# hooks (measured on 1.3.14), but it's timer-based: a PGLite WASM call
|
||||
# that blocks the event loop synchronously never lets the timer fire and
|
||||
# Hard outer timeout (180s per file). bun's --timeout is per-test; if a
|
||||
# PGLite WASM call hangs in beforeAll/afterAll, --timeout never fires and
|
||||
# the file wedges indefinitely. gtimeout/timeout SIGKILLs the file so the
|
||||
# suite advances. gtimeout (macOS via coreutils) preferred; timeout (Linux)
|
||||
# fallback; bare bun (no outer cap) if neither is installed.
|
||||
|
||||
@@ -64,7 +64,6 @@ CHECKS=(
|
||||
"check:source-scope-onboard"
|
||||
"check:no-double-retry"
|
||||
"check:batch-audit-site"
|
||||
"check:engine-dynamic-import"
|
||||
"check:worker-lock-renewal-shape"
|
||||
"typecheck"
|
||||
)
|
||||
|
||||
+13
-106
@@ -9,7 +9,7 @@ installSigchldHandler();
|
||||
import { installSignalHandlers as installCleanupSignalHandlers } from './core/process-cleanup.ts';
|
||||
installCleanupSignalHandlers();
|
||||
|
||||
import { readFileSync, existsSync, unlinkSync, fstatSync } from 'fs';
|
||||
import { readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { spawn } from 'child_process';
|
||||
import {
|
||||
readUpdateCache,
|
||||
@@ -55,7 +55,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector', 'backfill']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -344,11 +344,6 @@ async function main() {
|
||||
// them out of the engine try/catch is safe and unlocks routing.
|
||||
const params = parseOpArgs(op, subArgs);
|
||||
|
||||
// #3513: stdin fill moved out of parseOpArgs so a non-TTY stdin with no
|
||||
// piped input can't block the parse forever — the bounded read leaves the
|
||||
// param unset on timeout and the required-param check below fails fast.
|
||||
await applyStdinParam(op, params);
|
||||
|
||||
// v0.27.1 (`gbrain query --image <path>`): swap the `image` param from
|
||||
// a filesystem path into base64 bytes + mime. The op accepts base64; the
|
||||
// CLI accepts a path. Helper is exported so tests can exercise the
|
||||
@@ -809,99 +804,18 @@ export function parseOpArgs(op: Operation, args: string[]): Record<string, unkno
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* #3513: read stdin into an op's stdin-capable param without ever blocking
|
||||
* forever. The old inline `readFileSync(0)` in parseOpArgs assumed non-TTY
|
||||
* implies piped content; a non-TTY stdin with NO input (CI step, cron job,
|
||||
* agent harness holding an unwritten pipe open) blocked the read until kill.
|
||||
*
|
||||
* Strategy by fd kind (fstat):
|
||||
* - TTY: skip, as before (interactive input is not an op-param source).
|
||||
* - regular file / /dev/null / anything not a pipe or socket: readFileSync
|
||||
* returns without blocking (`gbrain put x < file`, `< /dev/null` → '').
|
||||
* - FIFO/socket: stream-read with a deadline on the FIRST byte only. A real
|
||||
* pipe (`echo foo | gbrain put x`, heredocs) delivers its first byte
|
||||
* within milliseconds; once any data arrives the deadline is lifted and
|
||||
* we read to EOF like readFileSync did (slow producers stay supported).
|
||||
* An empty-but-closed pipe (`: | gbrain put x`) EOFs immediately → ''.
|
||||
* A pipe that never delivers a byte times out → param stays unset, so
|
||||
* the existing required-param usage error fires (fail fast, exit 1).
|
||||
*
|
||||
* GBRAIN_STDIN_TIMEOUT_MS overrides the first-byte deadline (default 5000).
|
||||
* Exported for tests; called by the op dispatch right after parseOpArgs.
|
||||
*/
|
||||
export async function applyStdinParam(
|
||||
op: Operation,
|
||||
params: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
// Branch shape (stdin hint + missing param + `!process.stdin.isTTY` gate +
|
||||
// 5MB cap) is pinned by the R4 regression test for PR #1325's Windows fix
|
||||
// (test/cycle/regression-pr-wave-r1-r2-r4.test.ts) — keep the spelling.
|
||||
// Read stdin for content params
|
||||
if (op.cliHints?.stdin && !params[op.cliHints.stdin] && !process.stdin.isTTY) {
|
||||
const content = await readStdinBounded();
|
||||
if (content === null) return; // no input arrived — let the required-param check fail fast
|
||||
const stdinContent = readFileSync(0, 'utf-8');
|
||||
const MAX_STDIN = 5_000_000; // 5MB
|
||||
if (Buffer.byteLength(content, 'utf-8') > MAX_STDIN) {
|
||||
if (Buffer.byteLength(stdinContent, 'utf-8') > MAX_STDIN) {
|
||||
console.error(`Error: stdin content exceeds ${MAX_STDIN} bytes. Split into smaller inputs.`);
|
||||
process.exit(1);
|
||||
}
|
||||
params[op.cliHints.stdin] = content;
|
||||
params[op.cliHints.stdin] = stdinContent;
|
||||
}
|
||||
}
|
||||
|
||||
/** First-byte deadline for pipe/socket stdin (#3513). Env-overridable escape hatch. */
|
||||
function stdinFirstByteTimeoutMs(): number {
|
||||
const n = Number(process.env.GBRAIN_STDIN_TIMEOUT_MS);
|
||||
return Number.isFinite(n) && n > 0 ? n : 5000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the full stdin content, '' for a readable-but-empty stdin, or
|
||||
* null when stdin is a pipe/socket that never delivered a byte within the
|
||||
* first-byte deadline (or the fd is closed/unreadable).
|
||||
*/
|
||||
export async function readStdinBounded(): Promise<string | null> {
|
||||
let isPipeOrSocket: boolean;
|
||||
try {
|
||||
const st = fstatSync(0);
|
||||
isPipeOrSocket = st.isFIFO() || st.isSocket();
|
||||
} catch {
|
||||
return null; // closed/invalid fd — treat as no input
|
||||
}
|
||||
if (!isPipeOrSocket) {
|
||||
// Regular file redirect, /dev/null, etc. — read returns without blocking.
|
||||
try {
|
||||
return readFileSync(0, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return await new Promise<string | null>((resolve) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let gotData = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (!gotData) {
|
||||
process.stdin.destroy();
|
||||
resolve(null);
|
||||
}
|
||||
}, stdinFirstByteTimeoutMs());
|
||||
const finish = () => {
|
||||
clearTimeout(timer);
|
||||
resolve(Buffer.concat(chunks).toString('utf-8'));
|
||||
};
|
||||
process.stdin.on('data', (c: Buffer) => {
|
||||
if (!gotData) {
|
||||
gotData = true;
|
||||
clearTimeout(timer); // deadline applies to the FIRST byte only
|
||||
}
|
||||
chunks.push(c);
|
||||
});
|
||||
process.stdin.once('end', finish);
|
||||
process.stdin.once('error', finish);
|
||||
});
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -958,8 +872,7 @@ export function applyThinClientSourceScope(
|
||||
params.source_id = resolved;
|
||||
}
|
||||
|
||||
// Exported for tests (same import-safety contract as applyThinClientSourceScope).
|
||||
export async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
|
||||
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
@@ -971,21 +884,16 @@ export async function makeContext(engine: BrainEngine, params: Record<string, un
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
try {
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
} catch (err) {
|
||||
// #1712: an EXPLICIT --source that fails to resolve (invalid id, or a
|
||||
// source that doesn't exist) must error loudly — the blanket swallow
|
||||
// turned `--source __all__` and typos into a silent `default` scope,
|
||||
// which is how three bug reports became debugging sessions.
|
||||
if (explicit) throw err;
|
||||
// Ambient resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
// to the cross-source view (D16 back-compat path).
|
||||
sourceId = undefined;
|
||||
@@ -2540,7 +2448,6 @@ TOOLS
|
||||
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
|
||||
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
|
||||
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
|
||||
backfill <kind|list> v0.30.1: run a registered backfill (effective-date, ...)
|
||||
orphans [--json] [--count] Find pages with no inbound wikilinks
|
||||
salience [--days N] [--kind P] v0.29: pages ranked by emotional + activity salience
|
||||
anomalies [--since D] [--sigma N] v0.29: cohort-based statistical anomalies (tag, type)
|
||||
|
||||
+2
-12
@@ -4349,18 +4349,8 @@ export async function checkCycleFreshness(
|
||||
: `'${source.id}'`;
|
||||
const raw = source.config?.last_full_cycle_at;
|
||||
if (typeof raw !== 'string') {
|
||||
// #2540: WARN, not FAIL. This check iterates EVERY local_path source,
|
||||
// so on a multi-source install where only some vaults are cycled
|
||||
// (e.g. one nightly `gbrain dream --dir <vault>`), a never-cycled
|
||||
// sibling source turned doctor permanently red — which erodes the
|
||||
// check's signal until real staleness hides inside the noise (the
|
||||
// reporter's install masked genuinely stale sources for weeks this
|
||||
// way). "Never cycled" also fires on a source added minutes ago.
|
||||
// A source that HAS cycled and then went stale still escalates
|
||||
// through the warn/fail age thresholds below — that is the
|
||||
// regression signal this check exists for.
|
||||
issues.push(`Source ${display} has never completed a full cycle`);
|
||||
hasWarnings = true;
|
||||
hasFailures = true;
|
||||
continue;
|
||||
}
|
||||
const last = new Date(raw).getTime();
|
||||
@@ -4396,7 +4386,7 @@ export async function checkCycleFreshness(
|
||||
return {
|
||||
name: 'cycle_freshness',
|
||||
status: 'warn',
|
||||
message: `${issues.join('; ')}. Run \`gbrain dream --source <id>\` to cycle a source, or start \`gbrain autopilot\`.`,
|
||||
message: `${issues.join('; ')}.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
|
||||
+3
-50
@@ -19,26 +19,6 @@ import {
|
||||
} from '../core/pace-mode.ts';
|
||||
import { tryAcquireDbLock, type DbLockHandle } from '../core/db-lock.ts';
|
||||
import { embedBackfillLockId } from '../core/embed-backfill-lock.ts';
|
||||
import { wrapChunkTextsForStoredMode } from '../core/embedding-context.ts';
|
||||
import { titleTierCorpusGeneration } from '../core/contextual-retrieval-service.ts';
|
||||
import type { Page } from '../core/types.ts';
|
||||
|
||||
/**
|
||||
* #3507 — after a plain re-embed fully re-embedded a `per_chunk_synopsis`
|
||||
* page at the title-only tier (see wrapChunkTextsForStoredMode), restamp the
|
||||
* page's CR state to 'title' so `contextual_retrieval_mode` keeps describing
|
||||
* the vectors actually in the column. The reindex sweep restores the synopsis
|
||||
* tier later. No-op for every other mode.
|
||||
*/
|
||||
export async function restampIfDemotedToTitleTier(
|
||||
engine: BrainEngine,
|
||||
page: Pick<Page, 'contextual_retrieval_mode'> | null | undefined,
|
||||
slug: string,
|
||||
sourceId: string,
|
||||
): Promise<void> {
|
||||
if (page?.contextual_retrieval_mode !== 'per_chunk_synopsis') return;
|
||||
await engine.updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration());
|
||||
}
|
||||
|
||||
export interface EmbedOpts {
|
||||
/** Embed ALL pages (every chunk). */
|
||||
@@ -619,11 +599,7 @@ async function embedPage(
|
||||
return;
|
||||
}
|
||||
|
||||
// #3507: embed with the page's STORED wrapping convention (title-tier
|
||||
// contextual prefix when the page was embedded wrapped), not raw
|
||||
// chunk_text — otherwise a re-embed silently strips the contextual
|
||||
// prefixes the sync path applied. fenced_code chunks stay unwrapped.
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed), { abortSignal: signal });
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text), { abortSignal: signal });
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
@@ -646,9 +622,6 @@ async function embedPage(
|
||||
// such a page and then stamps it.
|
||||
if (toEmbed.length === chunks.length) {
|
||||
await engine.setPageEmbeddingSignature(slug, { sourceId, signature: currentEmbeddingSignature() });
|
||||
// #3507: a fully re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest.
|
||||
await restampIfDemotedToTitleTier(engine, page, slug, page.source_id);
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
@@ -790,8 +763,7 @@ async function embedAll(
|
||||
}
|
||||
|
||||
try {
|
||||
// #3507: reproduce the page's stored wrapping convention (see embedPage).
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed));
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text));
|
||||
// Build a map of new embeddings by chunk_index
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
@@ -813,11 +785,6 @@ async function embedAll(
|
||||
await observed(pacer, () =>
|
||||
engine.setPageEmbeddingSignature(page.slug, { sourceId: pageSourceId, signature }),
|
||||
);
|
||||
// #3507: --all fully re-embeds; a per_chunk_synopsis page landed at
|
||||
// the title tier — keep the stamped mode honest.
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, page, page.slug, pageSourceId),
|
||||
);
|
||||
result.embedded += toEmbed.length;
|
||||
} catch (e: unknown) {
|
||||
serr(`\n Error embedding ${page.slug}: ${e instanceof Error ? e.message : e}`);
|
||||
@@ -1131,13 +1098,7 @@ async function embedAllStale(
|
||||
const keySourceId = stale[0]?.source_id ?? 'default';
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes — `embed --stale` is the
|
||||
// NORMAL post-model-migration path, so raw-text embedding here
|
||||
// quietly converted whole corpora to the unwrapped convention.
|
||||
const pageRow = await observed(pacer, () => engine.getPage(slug, { sourceId: keySourceId }));
|
||||
const embeddings = await embedBatchWithBackoff(wrapChunkTextsForStoredMode(pageRow, stale), { abortSignal: effectiveSignal });
|
||||
const embeddings = await embedBatchWithBackoff(stale.map(c => c.chunk_text), { abortSignal: effectiveSignal });
|
||||
// Re-fetch existing chunks and merge to avoid deleting non-stale chunks.
|
||||
const existing = await observed(pacer, () => engine.getChunks(slug, { sourceId: keySourceId }));
|
||||
const staleIdxToEmbedding = new Map<number, Float32Array>();
|
||||
@@ -1165,14 +1126,6 @@ async function embedAllStale(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest. Partially-stale pages
|
||||
// stay stamped as-is (mixed provenance; reindex sweeps fix them).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
} catch (e: unknown) {
|
||||
// Budget/abort-fired cancellations are expected on the way out; don't
|
||||
|
||||
+2
-10
@@ -16,7 +16,7 @@ interface FileRecord {
|
||||
filename: string;
|
||||
storage_path: string;
|
||||
mime_type: string | null;
|
||||
size_bytes: number | bigint | string | null;
|
||||
size_bytes: number;
|
||||
content_hash: string;
|
||||
metadata: Record<string, unknown>;
|
||||
created_at: string;
|
||||
@@ -42,14 +42,6 @@ function fileHash(filePath: string): string {
|
||||
return createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
export function formatFileSizeKb(rawSizeBytes: number | bigint | string | null): string {
|
||||
if (rawSizeBytes == null) return '?';
|
||||
const sizeBytes = Number(rawSizeBytes);
|
||||
return Number.isFinite(sizeBytes) && sizeBytes >= 0
|
||||
? `${Math.round(sizeBytes / 1024)}KB`
|
||||
: '?';
|
||||
}
|
||||
|
||||
export async function runFiles(engine: BrainEngine, args: string[]) {
|
||||
const subcommand = args[0];
|
||||
|
||||
@@ -124,7 +116,7 @@ async function listFiles(engine: BrainEngine, slug?: string) {
|
||||
|
||||
console.log(`${rows.length} file(s):`);
|
||||
for (const row of rows) {
|
||||
const size = formatFileSizeKb(row.size_bytes as FileRecord['size_bytes']);
|
||||
const size = row.size_bytes ? `${Math.round(Number(row.size_bytes) / 1024)}KB` : '?';
|
||||
console.log(` ${row.page_slug || '(unlinked)'} / ${row.filename} [${size}, ${row.mime_type || '?'}]`);
|
||||
}
|
||||
}
|
||||
|
||||
+3
-10
@@ -233,7 +233,7 @@ USAGE
|
||||
gbrain jobs get <id>
|
||||
gbrain jobs cancel <id>
|
||||
gbrain jobs retry <id>
|
||||
gbrain jobs prune [--older-than 30d] [--dry-run]
|
||||
gbrain jobs prune [--older-than 30d]
|
||||
gbrain jobs delete <id>
|
||||
gbrain jobs stats
|
||||
gbrain jobs smoke
|
||||
@@ -633,15 +633,8 @@ HANDLER TYPES (built in)
|
||||
try { await queue.ensureSchema(); }
|
||||
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
|
||||
|
||||
// #2712: --dry-run previews the count without deleting. It used to be
|
||||
// silently ignored (the destructive default ran anyway).
|
||||
const dryRun = hasFlag(args, '--dry-run');
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000), dryRun });
|
||||
if (dryRun) {
|
||||
console.log(`[dry-run] Would prune ${count} jobs older than ${days} days. Nothing deleted.`);
|
||||
} else {
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
}
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000) });
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
|
||||
import express from 'express';
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import type { Server as HttpServer } from 'http';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import cors from 'cors';
|
||||
import rateLimit from 'express-rate-limit';
|
||||
@@ -47,7 +46,6 @@ import {
|
||||
type IngestionEvent,
|
||||
} from '../core/ingestion/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
|
||||
/**
|
||||
* /health endpoint timeout. 3s rather than 5s: Fly.io's default
|
||||
@@ -57,71 +55,6 @@ import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
*/
|
||||
export const HEALTH_TIMEOUT_MS = 3000;
|
||||
|
||||
/** Exported so tests can type their structural fakes exactly (#3599). */
|
||||
export type HttpServerLifecycle = Pick<HttpServer, 'listening' | 'once' | 'off' | 'close'>;
|
||||
/** Exported so tests can type their structural fakes exactly (#3599). */
|
||||
export type SignalSource = Pick<NodeJS.Process, 'once' | 'off'>;
|
||||
type CleanupRegistrar = typeof registerCleanup;
|
||||
|
||||
/**
|
||||
* Keep the HTTP server strongly referenced and make the daemon lifetime
|
||||
* explicit instead of relying on runtime-specific event-loop behavior for an
|
||||
* unobserved `app.listen()` return value. The shared abnormal-termination
|
||||
* cleanup pass closes it before process exit.
|
||||
*/
|
||||
export function waitForHttpServerLifecycle(
|
||||
server: HttpServerLifecycle,
|
||||
options: {
|
||||
signals?: SignalSource;
|
||||
register?: CleanupRegistrar;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const signals = options.signals ?? process;
|
||||
const register = options.register ?? registerCleanup;
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let closePromise: Promise<void> | null = null;
|
||||
|
||||
const closeServer = (): Promise<void> => {
|
||||
if (closePromise) return closePromise;
|
||||
closePromise = new Promise<void>((closeResolve, closeReject) => {
|
||||
if (!server.listening) {
|
||||
closeResolve();
|
||||
return;
|
||||
}
|
||||
server.close((error?: Error) => {
|
||||
if (error) closeReject(error);
|
||||
else closeResolve();
|
||||
});
|
||||
});
|
||||
return closePromise;
|
||||
};
|
||||
|
||||
const deregister = register('http-server', closeServer);
|
||||
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
server.off('close', onClose);
|
||||
server.off('error', onError);
|
||||
signals.off('SIGINT', onSigint);
|
||||
deregister();
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
};
|
||||
const onClose = () => finish();
|
||||
const onError = (error: Error) => finish(error);
|
||||
const onSigint = () => {
|
||||
void closeServer().catch(onError);
|
||||
};
|
||||
|
||||
server.once('close', onClose);
|
||||
server.once('error', onError);
|
||||
signals.once('SIGINT', onSigint);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.36.1.x #1024: bootstrap token resolution.
|
||||
*
|
||||
@@ -202,25 +135,6 @@ export type ProbeHealthResult =
|
||||
| { ok: true; status: 200; body: { status: 'ok'; version: string; engine: string; [k: string]: unknown } }
|
||||
| { ok: false; status: 503; body: { error: 'service_unavailable'; error_description: string } };
|
||||
|
||||
/** Exported so tests can type their structural fakes exactly (#3598). */
|
||||
export type AdminSseResponse = Pick<Response, 'setHeader' | 'flushHeaders' | 'write'>;
|
||||
|
||||
/**
|
||||
* Complete the admin EventSource handshake immediately.
|
||||
*
|
||||
* `flushHeaders()` alone can leave reverse proxies and browsers waiting for
|
||||
* the first response body bytes. An SSE comment is protocol-valid, ignored by
|
||||
* EventSource consumers, and makes the stream observable end-to-end without
|
||||
* fabricating an application event.
|
||||
*/
|
||||
export function openAdminSseStream(res: AdminSseResponse): void {
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.flushHeaders();
|
||||
res.write(': connected\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure async health probe. Races `engine.getStats()` against a timeout,
|
||||
* returns a tagged result. No Express coupling — easy to unit-test with a
|
||||
@@ -1718,7 +1632,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// SSE live activity feed
|
||||
// ---------------------------------------------------------------------------
|
||||
app.get('/admin/events', requireAdmin, (req: Request, res: Response) => {
|
||||
openAdminSseStream(res);
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.flushHeaders();
|
||||
|
||||
sseClients.add(res);
|
||||
req.on('close', () => sseClients.delete(res));
|
||||
@@ -2493,7 +2410,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// ---------------------------------------------------------------------------
|
||||
const clientCount = await sql`SELECT count(*)::int as count FROM oauth_clients`;
|
||||
|
||||
const httpServer = app.listen(port, bind, () => {
|
||||
app.listen(port, bind, () => {
|
||||
console.error(`
|
||||
╔══════════════════════════════════════════════════════╗
|
||||
║ GBrain MCP Server v${VERSION.padEnd(37)}║
|
||||
@@ -2518,6 +2435,4 @@ ${bootstrapFromEnv
|
||||
: `║ Admin Token (paste into /admin login): ║\n║ ${bootstrapToken.substring(0, 50)} ║\n║ ${bootstrapToken.substring(50).padEnd(50)} ║\n╚══════════════════════════════════════════════════════╝`}
|
||||
`);
|
||||
});
|
||||
|
||||
await waitForHttpServerLifecycle(httpServer);
|
||||
}
|
||||
|
||||
+4
-81
@@ -2874,17 +2874,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
: await resolveSlugByPathOrSourcePath(engine, from, undefined);
|
||||
// The new path doesn't yet have a row, so resolve from path only.
|
||||
const newSlug = resolveSlugForPath(to);
|
||||
// #3056: the cheap rename is OBSERVED, not assumed. A zero-row UPDATE
|
||||
// doesn't throw, and a thrown collision used to be swallowed by an
|
||||
// empty catch — both fell through to importFile, which created/updated
|
||||
// the row at the new path while the old row stayed behind live. Both
|
||||
// shapes now fall through to the reconcile below.
|
||||
let renameApplied = false;
|
||||
try {
|
||||
renameApplied = (await engine.updateSlug(oldSlug, newSlug, renameOpts)) > 0;
|
||||
await engine.updateSlug(oldSlug, newSlug, renameOpts);
|
||||
} catch {
|
||||
// Destination slug occupied or invalid — treat as add; the reconcile
|
||||
// below removes the stale old row once the destination materialized.
|
||||
// Slug doesn't exist or collision, treat as add
|
||||
}
|
||||
// Reimport at new path (picks up content changes). Wrapped to match the
|
||||
// deletes/adds loops: a malformed renamed file is recorded to failedFiles
|
||||
@@ -2897,11 +2890,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// NAV-1 TOCTOU: refuse a destination that realpath-resolves outside the
|
||||
// repo (committed symlink pointing out).
|
||||
const filePath = join(gitContextRoot, to);
|
||||
let importResult: Awaited<ReturnType<typeof importFile>> | undefined;
|
||||
if (existsSync(filePath) && isPathSafe(filePath, gitContextRoot)) {
|
||||
try {
|
||||
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
|
||||
importResult = result;
|
||||
if (result.status === 'imported') chunksCreated += result.chunks;
|
||||
else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
failedFiles.push({ path: to, error: String((result as { error?: string }).error) });
|
||||
@@ -2910,68 +2901,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
failedFiles.push({ path: to, error: e instanceof Error ? e.message : String(e) });
|
||||
}
|
||||
}
|
||||
// #3056 reconcile: the rename fell back to add semantics, so the row
|
||||
// that still represents the OLD path is the stale half of the rename
|
||||
// (git reported the old path gone; a plain delete of that path would
|
||||
// remove this row). Two safety rails, both from the #3252 review:
|
||||
//
|
||||
// 1. Delete only after the destination demonstrably materialized —
|
||||
// `imported`, or an errorless `skipped` AT the new slug. Identity
|
||||
// dedup can skip against the OLD row (result.slug === oldSlug),
|
||||
// in which case nothing landed at newSlug and deleting the old
|
||||
// row would destroy the only copy.
|
||||
// 2. Locate the stale row POSITIVELY by `source_path = from`, never
|
||||
// by the oldSlug guess — after a collision, a path-derived
|
||||
// fallback slug could name an unrelated (e.g. manually curated)
|
||||
// row. No source_path match → nothing is deleted (this also means
|
||||
// code-strategy imports, which don't populate source_path, fall
|
||||
// back safely to leaving the old row rather than guessing).
|
||||
//
|
||||
// A failed delete records a `<rename:…>` SENTINEL (not an ordinary
|
||||
// path failure): the gate hard-blocks the bookmark, and — unlike a
|
||||
// plain path row — the auto-skip valve can never chronic-skip it after
|
||||
// N attempts, which would advance the bookmark and make a transient
|
||||
// delete outage a permanent duplicate. The sentinel clears through the
|
||||
// ordinary success path once the rename converges on a later run.
|
||||
let reconcileFailed = false;
|
||||
if (!renameApplied && importResult !== undefined) {
|
||||
const destMaterialized = importResult.status === 'imported' ||
|
||||
(importResult.status === 'skipped' && !importResult.error && importResult.slug === newSlug);
|
||||
if (destMaterialized) {
|
||||
try {
|
||||
const staleMap = await engine.resolveSlugsByPaths([from], { sourceId: opts.sourceId ?? DEFAULT_SOURCE_ID });
|
||||
const staleSlug = staleMap.get(from);
|
||||
if (staleSlug !== undefined && staleSlug !== newSlug) {
|
||||
await engine.deletePage(staleSlug, renameOpts);
|
||||
deletedSlugs.add(staleSlug); // never hand a deleted slug to auto-embed
|
||||
serr(` [sync] rename reconciled: removed stale row ${staleSlug} (${from} -> ${to} fell back to add).`);
|
||||
} else if (staleSlug === undefined) {
|
||||
serr(` [sync] rename fallback: no row has source_path ${from}; stale row (if any) left in place.`);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
reconcileFailed = true;
|
||||
failedFiles.push({
|
||||
path: `<rename:${to}>`,
|
||||
error: `rename reconcile failed (stale row for ${from} not removed): ` +
|
||||
`${e instanceof Error ? e.message : String(e)}`,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
serr(
|
||||
` [sync] rename fallback: ${from} -> ${to} did not materialize at ${newSlug} ` +
|
||||
`(import ${importResult.status}); old row left in place.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Converged (cheap rename, clean reconcile, or nothing to reconcile):
|
||||
// clear any `<rename:…>` sentinel a previous failing run recorded.
|
||||
if (!reconcileFailed) succeededPaths.push(`<rename:${to}>`);
|
||||
pagesAffected.push(newSlug);
|
||||
deletedSlugs.delete(newSlug); // #1284: rename landed on a previously-deleted slug → embeddable again
|
||||
// A failed reconcile must NOT checkpoint: banking `to` would make the
|
||||
// resume filter skip this rename on the retry run, turning a transient
|
||||
// delete failure into a permanent duplicate — the exact bug being fixed.
|
||||
if (!reconcileFailed) await markCompleted(to);
|
||||
await markCompleted(to);
|
||||
progress.tick(1, newSlug);
|
||||
}
|
||||
progress.finish();
|
||||
@@ -3430,10 +3362,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
|
||||
if (!gate.advanced) {
|
||||
const codeBreakdown = formatCodeBreakdown(failedFiles);
|
||||
// Two sentinel classes block here: `<head>` (pin ancestry broken) and
|
||||
// `<rename:…>` (#3056 — a rename-reconcile delete failed and advancing
|
||||
// would permanently bank the duplicate). Pick the message by which fired.
|
||||
if (gate.sentinelBlocked && failedFiles.some(f => f.path === '<head>')) {
|
||||
if (gate.sentinelBlocked) {
|
||||
serr(
|
||||
`\nSync blocked: repository history changed during sync (force-push / reset).\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
@@ -3441,12 +3370,6 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
`a commit that doesn't match the indexed tree. Re-run sync to re-pin against ` +
|
||||
`current HEAD.`,
|
||||
);
|
||||
} else if (gate.sentinelBlocked) {
|
||||
serr(
|
||||
`\nSync blocked: a rename left a stale duplicate that could not be removed:\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
`The next 'gbrain sync' retries the reconcile from the same diff.`,
|
||||
);
|
||||
} else {
|
||||
const fileFailCount = failedFiles.filter(f => isSkippablePath(f.path)).length;
|
||||
serr(
|
||||
|
||||
+8
-15
@@ -3,7 +3,6 @@
|
||||
*
|
||||
* Subcommands:
|
||||
* takes <slug> — list takes for a page
|
||||
* takes list — list all active takes (#2079)
|
||||
* takes search "<query>" [--who h] — keyword search across all takes
|
||||
* takes add <slug> ...flags — append a take (markdown + DB)
|
||||
* takes update <slug> --row N ...flags — update mutable fields
|
||||
@@ -130,10 +129,11 @@ function writeBody(path: string, body: string): void {
|
||||
// --- Subcommands ---
|
||||
|
||||
async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
// #2079: slug is optional. `gbrain takes list` (no slug) lists ALL active
|
||||
// takes — CLI parity with the takes_list operation. A leading flag is not
|
||||
// a slug.
|
||||
const slug = args[0] && !args[0].startsWith('-') ? args[0] : undefined;
|
||||
const slug = args[0];
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain takes <slug> [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
const json = flagPresent(args, '--json');
|
||||
const holder = flagValue(args, '--who');
|
||||
const kind = flagValue(args, '--kind') as string | undefined;
|
||||
@@ -153,19 +153,17 @@ async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const scope = slug ?? 'this brain';
|
||||
if (takes.length === 0) {
|
||||
console.log(`No takes on ${scope}.`);
|
||||
console.log(`No takes on ${slug}.`);
|
||||
return;
|
||||
}
|
||||
console.log(`# Takes on ${scope}\n`);
|
||||
console.log(`# Takes on ${slug}\n`);
|
||||
for (const t of takes) {
|
||||
const tag = t.active ? '' : ' [superseded]';
|
||||
const w = Number(t.weight).toFixed(2);
|
||||
const since = t.since_date ?? '';
|
||||
const src = t.source ? ` — ${t.source}` : '';
|
||||
const where = slug ? '' : `${t.page_slug} `;
|
||||
console.log(`${where}#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
console.log(`#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -557,8 +555,6 @@ export async function runTakes(engine: BrainEngine, args: string[]): Promise<voi
|
||||
Subcommands:
|
||||
takes <slug> [--json] [--who h] [--kind k] [--sort weight|since_date|created_at] [--expired]
|
||||
List takes for a page
|
||||
takes list [--json] [--who h] [--kind k] [--sort ...] [--expired]
|
||||
List all active takes across the brain (#2079)
|
||||
takes search "<query>" [--limit N] [--json]
|
||||
Keyword search across all takes
|
||||
takes add <slug> --claim "..." --kind <fact|take|bet|hunch> --who <holder>
|
||||
@@ -588,9 +584,6 @@ Common flags:
|
||||
const rest = args.slice(1);
|
||||
|
||||
switch (sub) {
|
||||
// #2079: `takes list` used to be parsed as page slug "list" and printed
|
||||
// "No takes on list." — reading exactly like an empty takes table.
|
||||
case 'list': return cmdList(engine, rest);
|
||||
case 'search': return cmdSearch(engine, rest);
|
||||
case 'add': return cmdAdd(engine, rest, await resolveTakesSourceId(engine));
|
||||
case 'update': return cmdUpdate(engine, rest, await resolveTakesSourceId(engine));
|
||||
|
||||
+2
-63
@@ -642,42 +642,8 @@ function warnRecipesMissingBatchTokens(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only reset baseline (#3554). The bunfig preload
|
||||
* (`test/helpers/legacy-embedding-preload.ts`) pins the gateway to the legacy
|
||||
* OpenAI/1536 config at process start, but `resetGateway()` used to wipe that
|
||||
* pin to `_config = null`. The next test file's engine connect then
|
||||
* reconfigured from the SHIPPED default (zembed-1 @ 1280) and every 1536-d
|
||||
* fixture in that file exploded with `expected 1280 dimensions, not 1536` —
|
||||
* a cross-file mine whose placement depended on shard bin-packing.
|
||||
*
|
||||
* When a baseline factory is registered, `resetGateway()` means "back to the
|
||||
* test baseline" instead of "unconfigured": it clears everything as before,
|
||||
* then re-applies the factory's config via `configureGateway()`. A factory
|
||||
* (not a frozen config) so each re-application captures fresh
|
||||
* `process.env`, matching the preload's original `applyLegacy()` semantics.
|
||||
*
|
||||
* Production is untouched: nothing in `src/` calls `resetGateway()` or this
|
||||
* setter, so in production the baseline is never registered and
|
||||
* `resetGateway()` still fully unconfigures. Same `__*ForTests` seam
|
||||
* convention as `__setEmbedTransportForTests` above.
|
||||
*/
|
||||
let _resetBaseline: (() => AIGatewayConfig) | null = null;
|
||||
|
||||
/**
|
||||
* Register (or clear, with `null`) the config factory that `resetGateway()`
|
||||
* re-applies. Called once by the bunfig test preload.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __setGatewayResetBaselineForTests(
|
||||
factory: (() => AIGatewayConfig) | null,
|
||||
): void {
|
||||
_resetBaseline = factory;
|
||||
}
|
||||
|
||||
/** Clear every piece of module state. Shared by both reset flavors. */
|
||||
function clearGatewayState(): void {
|
||||
/** Reset (for tests). */
|
||||
export function resetGateway(): void {
|
||||
_config = null;
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
@@ -689,33 +655,6 @@ function clearGatewayState(): void {
|
||||
_extendedModels.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset (for tests). Clears all module state (config, model cache, shrink
|
||||
* state, transports, warned recipes, extended models), then — if a test
|
||||
* baseline is registered — re-applies it so the gateway returns to the
|
||||
* process-wide test default instead of an unconfigured limbo (#3554).
|
||||
*/
|
||||
export function resetGateway(): void {
|
||||
clearGatewayState();
|
||||
// configureGateway re-clears _modelCache/_shrinkState/_extendedModels and
|
||||
// registers the baseline's models; transports are NOT touched by it, so a
|
||||
// stale test transport can never leak back in through this path.
|
||||
if (_resetBaseline) configureGateway(_resetBaseline());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset AND stay unconfigured, ignoring any registered baseline. For the
|
||||
* handful of tests that assert genuine no-gateway behavior
|
||||
* (`no_gateway_config` diagnosis, `isAvailable() === false`, graceful
|
||||
* degradation paths). The preload's per-test beforeEach restores the
|
||||
* baseline before the next test, so this cannot leak across tests.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __unconfigureGatewayForTests(): void {
|
||||
clearGatewayState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only seam. Replaces the function the gateway calls to embed a
|
||||
* sub-batch. Pass `null` to restore the real `embedMany` from the AI SDK.
|
||||
|
||||
+5
-28
@@ -21,35 +21,12 @@ export const CJK_SLUG_CHARS = '一-鿿-ゟ゠-ヿ가-';
|
||||
export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`);
|
||||
|
||||
/**
|
||||
* Slug "word" character class (#3417): every script's letters, not just
|
||||
* Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any
|
||||
* regex composed from this string (without `u`, `\p{Ll}` silently matches
|
||||
* the literal chars `p`, `L`, `l`, `{`, `}`).
|
||||
*
|
||||
* \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …)
|
||||
* \p{Lm} modifier letters
|
||||
* \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …)
|
||||
* \p{M} combining marks that survive the Latin accent-strip pass
|
||||
* (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs)
|
||||
* \p{N} numbers (0-9, Arabic-Indic digits, …)
|
||||
*
|
||||
* Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment()
|
||||
* lowercases before filtering, so validators stay lowercase-canonical.
|
||||
*
|
||||
* Distinct from CJK_SLUG_CHARS above, which also drives the
|
||||
* countCJKAwareWords density heuristic — do NOT merge the two, or slug
|
||||
* grammar changes silently change chunking behavior.
|
||||
* Page-slug segment grammar (no anchors): alnum-or-CJK lead char, then
|
||||
* alnum/CJK/hyphen continuation. Single source for validatePageSlug
|
||||
* (operations.ts), SlugRegistry's SLUG_RE, and the dream-cycle
|
||||
* SUMMARY_SLUG_RE so every slug validator shares one grammar (#738).
|
||||
*/
|
||||
export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}';
|
||||
|
||||
/**
|
||||
* Page-slug segment grammar (no anchors): word-char lead, then word-char or
|
||||
* hyphen continuation. Single source for validatePageSlug (operations.ts),
|
||||
* SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug
|
||||
* validator shares one grammar (#738). Compose with the `u` flag — see
|
||||
* SLUG_WORD_CHARS.
|
||||
*/
|
||||
export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`;
|
||||
export const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`;
|
||||
|
||||
export const CJK_SENTENCE_DELIMITERS = ['。', '!', '?']; // 。!?
|
||||
export const CJK_CLAUSE_DELIMITERS = [';', ':', ',', '、']; // ;:,、
|
||||
|
||||
@@ -145,19 +145,6 @@ export function computeCorpusGeneration(args: {
|
||||
return h.digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — the corpus_generation a page lands on when a plain re-embed path
|
||||
* (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
|
||||
* title-only tier (the D14 fallback tier; synopsis re-generation is a paid
|
||||
* backfill concern). Callers restamp
|
||||
* `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
|
||||
* so the stamped mode keeps describing the vectors actually in the column.
|
||||
* Matches what the inline import path writes for its title-tier pages.
|
||||
*/
|
||||
export function titleTierCorpusGeneration(): string {
|
||||
return computeCorpusGeneration({ crMode: 'title', haikuModel: DEFAULT_HAIKU_MODEL });
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute source_text_hash for D27 P1-4 cache key composition. The
|
||||
* synopsis cache invalidates correctly when adjacent text changes (page
|
||||
|
||||
+2
-13
@@ -895,17 +895,8 @@ export async function resolveSourceForDir(
|
||||
// (the cycleSourceId precedence) or 'default'.
|
||||
if (brainDir === null) return undefined;
|
||||
try {
|
||||
// #2540: exclude archived rows (dream's --source guard refuses to stamp
|
||||
// them, so an archived alias winning here means the stamp silently never
|
||||
// lands and doctor's cycle_freshness stays red on a healthy install) and
|
||||
// order deterministically so a duplicate registration of the same path
|
||||
// can't shadow the active source on whichever row the engine scans first.
|
||||
// Ordering matches listAllSources/sources-ops for operator-output parity.
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM sources
|
||||
WHERE local_path = $1 AND archived = false
|
||||
ORDER BY (id = 'default') DESC, id
|
||||
LIMIT 1`,
|
||||
`SELECT id FROM sources WHERE local_path = $1 LIMIT 1`,
|
||||
[brainDir],
|
||||
);
|
||||
if (rows[0]) return rows[0].id;
|
||||
@@ -1188,9 +1179,7 @@ async function runPhaseExtractFacts(
|
||||
summary: `extract_facts skipped: ${result.legacyRowsPending} legacy v0.31 facts pending fence backfill`,
|
||||
details: {
|
||||
legacyRowsPending: result.legacyRowsPending,
|
||||
// A bare `apply-migrations --yes` no-ops once the v0.32.2 ledger
|
||||
// entry is complete; the retry marker is what re-runs Phase B.
|
||||
hint: 'gbrain apply-migrations --force-retry 0.32.2 && gbrain apply-migrations --yes',
|
||||
hint: 'gbrain apply-migrations --yes',
|
||||
warnings: result.warnings,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -26,17 +26,13 @@
|
||||
*
|
||||
* Empty-fence guard (Codex R2-#7; #2484; #2646): the phase refuses to do
|
||||
* its destructive reconciliation pass when genuinely-backfillable legacy
|
||||
* rows still exist — in THIS run's source only (`source_id = sourceId`;
|
||||
* a pending row in source A must not jam extraction for source B — the
|
||||
* source-isolation invariant) — `row_num IS NULL` (never fenced) AND
|
||||
* `entity_slug` resolves to a live page in this source (so the v0_32_2
|
||||
* migration's Phase B could fence them) AND the row is not soft-expired
|
||||
* (`expired_at IS NULL`). Status returns `warn` with a hint to re-run
|
||||
* the v0.32.2 fence backfill (`apply-migrations --force-retry 0.32.2`
|
||||
* then `--yes` — a bare `--yes` is a no-op once the ledger says
|
||||
* complete). Without the guard, an interrupted upgrade where v0_32_2
|
||||
* hasn't run could leave the cycle silently misreporting "0 facts on
|
||||
* people/alice" while legacy rows linger.
|
||||
* rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug`
|
||||
* resolves to a live page in this source (so the v0_32_2 migration's
|
||||
* Phase B could fence them) AND the row is not soft-expired
|
||||
* (`expired_at IS NULL`). Status returns `warn` with a hint to run
|
||||
* `gbrain apply-migrations --yes`. Without the guard, an interrupted
|
||||
* upgrade where v0_32_2 hasn't run could leave the cycle silently
|
||||
* misreporting "0 facts on people/alice" while legacy rows linger.
|
||||
*
|
||||
* The live-page requirement (#2484) is load-bearing: the inline facts
|
||||
* writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL`
|
||||
@@ -229,17 +225,10 @@ export async function runExtractFacts(
|
||||
// soft-expires legacy rows rather than deleting them, so counting
|
||||
// expired rows would leave the guard permanently stuck with no
|
||||
// supported way to drain the backlog.
|
||||
//
|
||||
// Source isolation (#3526): the count is scoped to THIS run's
|
||||
// sourceId. The pre-fix query counted brain-wide, so a single pending
|
||||
// legacy row in any mounted source jammed extract_facts for every
|
||||
// source — a cross-source leak of one source's migration state into
|
||||
// another's cycle (CLAUDE.md source-isolation invariant).
|
||||
const legacy = await engine.executeRaw<{ n: string }>(
|
||||
`SELECT COUNT(*) AS n
|
||||
FROM facts f
|
||||
WHERE f.source_id = $1
|
||||
AND f.row_num IS NULL
|
||||
WHERE f.row_num IS NULL
|
||||
AND f.entity_slug IS NOT NULL
|
||||
AND f.expired_at IS NULL
|
||||
AND EXISTS (
|
||||
@@ -248,25 +237,15 @@ export async function runExtractFacts(
|
||||
AND p.slug = f.entity_slug
|
||||
AND p.deleted_at IS NULL
|
||||
)`,
|
||||
[sourceId],
|
||||
);
|
||||
const legacyCount = parseInt(legacy[0]?.n ?? '0', 10);
|
||||
result.legacyRowsPending = legacyCount;
|
||||
if (legacyCount > 0) {
|
||||
result.guardTriggered = true;
|
||||
// Drain advice must actually work: a bare `apply-migrations --yes`
|
||||
// is a no-op once the v0.32.2 ledger entry says complete (the
|
||||
// runner classifies it as already-applied), so the sanctioned
|
||||
// re-run path is the explicit retry marker first. Phase B is
|
||||
// idempotent — it only touches `row_num IS NULL` rows and de-dupes
|
||||
// against the existing fence — so the re-run is safe. Individual
|
||||
// rows can instead be drained through `forget_fact` (soft-expired
|
||||
// rows stop counting).
|
||||
result.warnings.push(
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows in source "${sourceId}" ` +
|
||||
`(entity page present, not yet fenced) pending fence backfill. Re-run the v0.32.2 ` +
|
||||
`fence backfill: \`gbrain apply-migrations --force-retry 0.32.2\` then ` +
|
||||
`\`gbrain apply-migrations --yes\`. Or drain individual rows via \`forget_fact\`.`,
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` +
|
||||
`fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` +
|
||||
`v0_32_2 before this phase can safely reconcile fence → DB.`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -48,9 +48,8 @@ import { safeSplitIndex } from '../text-safe.ts';
|
||||
import { PAGE_SLUG_SEG } from '../cjk.ts';
|
||||
|
||||
// Slug grammar from validatePageSlug — shared via PAGE_SLUG_SEG (#738).
|
||||
// Used for the orchestrator-written summary index slug. `u` flag required
|
||||
// by PAGE_SLUG_SEG's \p{...} classes (#3417).
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'u');
|
||||
// Used for the orchestrator-written summary index slug.
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`);
|
||||
|
||||
// ── Model context budget (D1, D5, D7, D9) ─────────────────────────────
|
||||
|
||||
|
||||
+2
-17
@@ -19,8 +19,7 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import { embedBatchWithBackoff, restampIfDemotedToTitleTier } from '../commands/embed.ts';
|
||||
import { wrapChunkTextsForStoredMode } from './embedding-context.ts';
|
||||
import { embedBatchWithBackoff } from '../commands/embed.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
|
||||
@@ -190,15 +189,8 @@ export async function embedStaleForSource(
|
||||
const keySourceId = stale[0]?.source_id ?? sourceId;
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes (mirrors
|
||||
// src/commands/embed.ts:embedAllStale).
|
||||
const pageRow = await observed(pacer, () =>
|
||||
engine.getPage(slug, { sourceId: keySourceId }),
|
||||
);
|
||||
const embeddings = await embedFn(
|
||||
wrapChunkTextsForStoredMode(pageRow, stale),
|
||||
stale.map((c) => c.chunk_text),
|
||||
{ abortSignal: signal },
|
||||
);
|
||||
const existing = await observed(pacer, () =>
|
||||
@@ -241,13 +233,6 @@ export async function embedStaleForSource(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest (mixed pages stay as-is).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
result.pagesProcessed += 1;
|
||||
} catch (e: unknown) {
|
||||
|
||||
@@ -186,41 +186,3 @@ export function modeRequiresHaiku(mode: CRMode): boolean {
|
||||
export function modeRequiresWrapper(mode: CRMode): boolean {
|
||||
return mode !== 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — build the embedding inputs for a re-embed of EXISTING chunk rows,
|
||||
* reproducing the wrapping convention the page's vectors were originally
|
||||
* built under (recorded in `pages.contextual_retrieval_mode`).
|
||||
*
|
||||
* Used by every plain re-embed path (`embed <slug>`, `embed --all`,
|
||||
* `embed --stale`, the embed-backfill Minion loop). Before this helper those
|
||||
* paths embedded raw `chunk_text`, so any re-embed — including the NORMAL
|
||||
* post-model-migration `embed --stale` — silently replaced context-wrapped
|
||||
* vectors with unwrapped ones, degrading retrieval with no signature change
|
||||
* to show for it.
|
||||
*
|
||||
* Convention rules (embed PRESERVES conventions; changing them is
|
||||
* sync/reindex's job):
|
||||
* - mode NULL/undefined/'none' → raw chunk_text (status quo).
|
||||
* - mode 'title' → title-only prefix (pure string concat).
|
||||
* - mode 'per_chunk_synopsis' → title-only prefix. Re-generating Haiku
|
||||
* synopses is a paid backfill concern; title-only is the service's own
|
||||
* documented fallback tier (D14). Callers that fully re-embed a page
|
||||
* this way should restamp the page to 'title' so the column stays
|
||||
* honest (see contextual-retrieval-service.ts:titleTierCorpusGeneration).
|
||||
* - `fenced_code` chunks are NEVER wrapped (D20-T4), same as sync.
|
||||
*/
|
||||
export function wrapChunkTextsForStoredMode(
|
||||
page:
|
||||
| { title?: string | null; contextual_retrieval_mode?: CRMode | null }
|
||||
| null
|
||||
| undefined,
|
||||
chunks: ReadonlyArray<{ chunk_text: string; chunk_source?: string | null }>,
|
||||
): string[] {
|
||||
const mode = page?.contextual_retrieval_mode;
|
||||
if (mode == null || !modeRequiresWrapper(mode)) {
|
||||
return chunks.map((c) => c.chunk_text);
|
||||
}
|
||||
const prefix = buildContextualPrefix(page?.title ?? '', null);
|
||||
return chunks.map((c) => wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source));
|
||||
}
|
||||
|
||||
+1
-6
@@ -1951,13 +1951,8 @@ export interface BrainEngine {
|
||||
* preserved via stable page_id). `opts.sourceId` scopes the UPDATE — without
|
||||
* it, the bare `WHERE slug = old` matches every row across every source and
|
||||
* would either rename them all OR violate the (source_id, slug) UNIQUE.
|
||||
*
|
||||
* Returns the number of rows moved. 0 means the old slug had no row in the
|
||||
* scoped source — an UPDATE that matches nothing does NOT throw, so callers
|
||||
* that need to know whether the rename actually happened (the sync rename
|
||||
* path, #3056) must check the return value rather than rely on the catch.
|
||||
*/
|
||||
updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number>;
|
||||
updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void>;
|
||||
rewriteLinks(oldSlug: string, newSlug: string): Promise<void>;
|
||||
|
||||
/**
|
||||
|
||||
@@ -85,21 +85,11 @@ const RUN_ID_SHORT_LEN = 8;
|
||||
/**
|
||||
* Truncate a run id to the standard 8-char short form used in slug
|
||||
* paths. Idempotent — passing an already-short id returns it unchanged.
|
||||
* Non-hex / non-alphanumeric chars survive INSIDE the short form
|
||||
* (op-checkpoint ids may include dashes or other separators), but
|
||||
* boundary hyphens are trimmed (#3443): `slugifySegment()` strips
|
||||
* leading/trailing hyphens during repo sync, so a short form like
|
||||
* 'propose-' (from propose-<timestamp> run ids) made the DB receipt
|
||||
* slug and its Git-backed slug disagree — writing the receipt through
|
||||
* to the repo created a normalized sibling instead of materializing
|
||||
* the existing page. Invariant: slugifySegment(shortRunId(x)) ===
|
||||
* shortRunId(x) for slug-safe run ids.
|
||||
* Non-hex / non-alphanumeric chars survive (op-checkpoint ids may
|
||||
* include dashes or other separators).
|
||||
*/
|
||||
export function shortRunId(runId: string): string {
|
||||
// ponytail: truncation-based discrimination is only as good as the run id's
|
||||
// first 8 chars; families that need per-run uniqueness must front-load it.
|
||||
const short = runId.slice(0, RUN_ID_SHORT_LEN).replace(/^-+|-+$/g, '');
|
||||
return short || (runId ? 'run' : '');
|
||||
return runId.slice(0, RUN_ID_SHORT_LEN);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1092,8 +1092,8 @@ export async function importFromFile(
|
||||
chunks: 0,
|
||||
error:
|
||||
`Filename "${relativePath}" produces no usable slug. ` +
|
||||
`Add a "slug:" to the frontmatter, or rename the file to include ` +
|
||||
`at least one letter or number (any script).`,
|
||||
`Add a "slug:" to the frontmatter, or rename the file to use ` +
|
||||
`ASCII / Chinese / Japanese / Korean characters.`,
|
||||
};
|
||||
}
|
||||
} else if (parsed.slug !== expectedSlug) {
|
||||
|
||||
+2
-7
@@ -2,13 +2,6 @@ import type { BrainEngine } from './engine.ts';
|
||||
import { slugifyPath } from './sync.ts';
|
||||
import { getFtsLanguage } from './fts-language.ts';
|
||||
import { hnswMaxDimsForType } from './vector-index.ts';
|
||||
// runMigrations executes while an initialized engine is live. Keep its helper
|
||||
// modules in the static graph rather than importing them from async handlers.
|
||||
import {
|
||||
isStatementTimeoutError,
|
||||
isRetryableConnError,
|
||||
} from './retry-matcher.ts';
|
||||
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
|
||||
|
||||
/**
|
||||
* Schema migrations — run automatically on initSchema().
|
||||
@@ -5808,6 +5801,7 @@ async function runMigrationSQLWithRetry(
|
||||
m: Migration,
|
||||
sql: string,
|
||||
): Promise<void> {
|
||||
const { isStatementTimeoutError, isRetryableConnError } = await import('./retry-matcher.ts');
|
||||
// GBRAIN_MIGRATE_BACKOFF_MS lets tests skip the 5s/15s/45s backoff. In
|
||||
// production the env var is unset and the default cadence applies.
|
||||
const fastBackoff = process.env.GBRAIN_MIGRATE_BACKOFF_MS;
|
||||
@@ -6077,6 +6071,7 @@ export async function runMigrations(engine: BrainEngine): Promise<{ applied: num
|
||||
// reach the loop below). Best-effort + idempotent: a no-op on a healthy
|
||||
// index; `doctor` surfaces it independently if this ever fails.
|
||||
try {
|
||||
const { repairTimelineDedupIndex } = await import('./timeline-dedup-repair.ts');
|
||||
const r = await repairTimelineDedupIndex(engine);
|
||||
if (r.repaired) {
|
||||
console.error(
|
||||
|
||||
@@ -43,11 +43,6 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
|
||||
// few writes. Generous 10-min budget (vs the tight null-default) covers a
|
||||
// slow gateway without the 30-min loop budget.
|
||||
chronicle_extract: TEN_MIN_MS,
|
||||
// #3207 — same shape as chronicle_extract: one page = one LLM extraction
|
||||
// call + a few writes. Was missing from this map, so it inherited the tight
|
||||
// null-default and got dead-lettered mid-generation on slow chat providers
|
||||
// (facts silently lost) — exactly the failure this file exists to prevent.
|
||||
'facts-absorb': TEN_MIN_MS,
|
||||
// Per-page contextual reindex jobs process chunks sequentially with one
|
||||
// rate-leased LLM synopsis call per chunk; large transcript pages need more
|
||||
// than the standard 30-min long-job budget.
|
||||
|
||||
@@ -534,21 +534,10 @@ export class MinionQueue {
|
||||
}
|
||||
|
||||
/** Prune old jobs in terminal statuses. Returns count of deleted rows. */
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[]; dryRun?: boolean }): Promise<number> {
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[] }): Promise<number> {
|
||||
const statuses = opts?.status ?? ['completed', 'dead', 'cancelled'];
|
||||
const olderThan = opts?.olderThan ?? new Date(Date.now() - 30 * 86400000);
|
||||
|
||||
// #2712: dryRun counts the would-be-pruned rows without deleting.
|
||||
// Silent-ignoring a safety flag on a delete path is data loss.
|
||||
if (opts?.dryRun) {
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text as count FROM minion_jobs
|
||||
WHERE status = ANY($1) AND updated_at < $2`,
|
||||
[statuses, olderThan.toISOString()]
|
||||
);
|
||||
return parseInt(rows[0]?.count ?? '0', 10);
|
||||
}
|
||||
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`WITH pruned AS (
|
||||
DELETE FROM minion_jobs
|
||||
|
||||
+5
-15
@@ -28,7 +28,6 @@ import { isSearchMode } from './search/mode.ts';
|
||||
import { stampEvidence } from './search/evidence.ts';
|
||||
import type { SearchResult } from './types.ts';
|
||||
import { CJK_SLUG_CHARS, PAGE_SLUG_SEG } from './cjk.ts';
|
||||
import { ALL_SOURCES } from './source-id.ts';
|
||||
import * as db from './db.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import {
|
||||
@@ -163,11 +162,10 @@ export function validatePageSlug(slug: string): void {
|
||||
if (slug.length > 255) {
|
||||
throw new OperationError('invalid_params', 'page_slug exceeds 255 characters');
|
||||
}
|
||||
// #3417: letters/numbers from any script allowed in segments (u flag required
|
||||
// for the \p{...} classes in PAGE_SLUG_SEG). Shape rules (lead char, hyphen
|
||||
// continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'iu').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: letters/numbers in any script, hyphens, forward-slash separated segments)`);
|
||||
// v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed
|
||||
// in segments. ASCII shape rules (lead char, hyphen continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -488,14 +486,6 @@ export function sourceScopeOpts(ctx: OperationContext): { sourceId?: string; sou
|
||||
// value of `[]` MUST NOT widen scope to "all sources" by being interpreted
|
||||
// as "no filter."
|
||||
if (allowed && allowed.length > 0) return { sourceIds: allowed };
|
||||
// #1712: the __all__ sentinel spans the brain — but ONLY for trusted local
|
||||
// callers (strictly `remote === false`). For remote/untrusted callers the
|
||||
// literal stays as-is: it can never match a real source id (underscores are
|
||||
// rejected at creation), so the read fail-closes to empty rather than
|
||||
// widening past the caller's grant. Do NOT "simplify" this to `{}`.
|
||||
if (ctx.sourceId === ALL_SOURCES) {
|
||||
return ctx.remote === false ? {} : { sourceId: ctx.sourceId };
|
||||
}
|
||||
if (ctx.sourceId) return { sourceId: ctx.sourceId };
|
||||
return {};
|
||||
}
|
||||
@@ -563,7 +553,7 @@ export function resolveRequestedScope(
|
||||
sourceIdParam: string | undefined,
|
||||
allSourcesParam = false,
|
||||
): { sourceId?: string; sourceIds?: string[] } {
|
||||
const wantsAll = allSourcesParam || sourceIdParam === ALL_SOURCES;
|
||||
const wantsAll = allSourcesParam || sourceIdParam === '__all__';
|
||||
if (wantsAll) {
|
||||
return ctx.remote === false ? {} : sourceScopeOpts(ctx);
|
||||
}
|
||||
|
||||
@@ -72,10 +72,9 @@ export class SlugRegistryError extends Error {
|
||||
// SlugRegistry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Shares the page-slug segment grammar (all scripts, #738/#3417) with
|
||||
// Shares the page-slug segment grammar (incl. CJK ranges, #738) with
|
||||
// validatePageSlug; keeps this site's dir/name shape (>= 2 segments).
|
||||
// `u` flag required by PAGE_SLUG_SEG's \p{...} classes.
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u');
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`);
|
||||
|
||||
export class SlugRegistry {
|
||||
constructor(private engine: BrainEngine) {}
|
||||
|
||||
+19
-71
@@ -17,26 +17,7 @@ import type {
|
||||
SourceRow,
|
||||
} from './engine.ts';
|
||||
import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts';
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
|
||||
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { PGLITE_SCHEMA_SQL, getPGLiteSchema } from './pglite-schema.ts';
|
||||
@@ -438,13 +419,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
let dims: number = DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
|
||||
await this.db.exec(getPGLiteSchema(dims, model));
|
||||
@@ -1002,8 +979,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
|
||||
params
|
||||
);
|
||||
@@ -2285,6 +2261,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.name === 'RetryAbortError') throw err;
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
if (isRetryableConnError(err)) {
|
||||
auditLogBatchExhausted(auditSite, batchSize, opts.maxRetries + 1, err);
|
||||
}
|
||||
@@ -2343,28 +2320,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
// Provenance fallback for chunks without an explicit `model`: resolve the
|
||||
// gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL.
|
||||
// #3461: getEmbeddingModel() THROWS when unconfigured (never returns
|
||||
// falsy) — on the throw path fall back to the brain's own
|
||||
// `config.embedding_model` row, then the compile-time default as the
|
||||
// last resort. See postgres-engine.ts _upsertChunksOnce for the full
|
||||
// rationale — pglite mirrors it for parity.
|
||||
let resolvedModel: string | null = null;
|
||||
// See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite
|
||||
// mirrors it for parity.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
} catch {
|
||||
try {
|
||||
const cfg = await this.db.query(
|
||||
`SELECT value FROM config WHERE key = 'embedding_model'`,
|
||||
);
|
||||
resolvedModel = ((cfg.rows[0] as { value?: string } | undefined)?.value) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2417,9 +2381,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch so the label always
|
||||
// describes whichever vector wins the upsert. See postgres-engine.ts for rationale.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2433,14 +2394,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
@@ -3864,6 +3818,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
async mergeOntologyFact(obs: OntologyObservationInput): Promise<OntologyMergeResult> {
|
||||
const sourceId = obs.sourceId ?? 'default';
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const dimension = normalizeDimension(obs.dimension);
|
||||
const vh = valueHash(obs.value);
|
||||
const conf = obs.confidence ?? 0.7;
|
||||
@@ -5353,16 +5308,12 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// pages_with_timeline) and v0.10.3 graph layer (link_coverage, timeline_coverage,
|
||||
// most_connected). Both coexist: master's brain_score is the composite
|
||||
// dashboard, v0.10.3 metrics give entity-page-level granularity.
|
||||
// #1305: every page-scoped count here excludes soft-deleted rows — same
|
||||
// posture as getStats — so brain_score moves when the user deletes pages.
|
||||
// Chunk/link counts stay raw (storage until the purge phase), matching
|
||||
// getStats, and destructive-removal counts elsewhere deliberately stay raw.
|
||||
const { rows: [h] } = await this.db.query(`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
@@ -5387,7 +5338,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL
|
||||
WHERE p.type IN ('entity', 'person', 'company')
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`);
|
||||
@@ -5406,7 +5357,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded,
|
||||
EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline
|
||||
FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
`);
|
||||
|
||||
const r = h as Record<string, unknown>;
|
||||
@@ -5501,18 +5451,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Sync
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number> {
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void> {
|
||||
newSlug = validateSlug(newSlug);
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// Source-qualify so a rename in source A doesn't sweep up same-slug rows
|
||||
// in sources B/C/D (mirrors postgres-engine.ts).
|
||||
const result = await this.db.query(
|
||||
await this.db.query(
|
||||
`UPDATE pages SET slug = $1, updated_at = now() WHERE slug = $2 AND source_id = $3`,
|
||||
[newSlug, oldSlug, sourceId]
|
||||
);
|
||||
// #3056: rows moved — a zero-row UPDATE does not throw, so the count is
|
||||
// the only way callers can see the no-op.
|
||||
return result.affectedRows ?? 0;
|
||||
}
|
||||
|
||||
async rewriteLinks(_oldSlug: string, _newSlug: string): Promise<void> {
|
||||
@@ -6026,6 +5973,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const recencyBias = opts.recency_bias ?? 'flat';
|
||||
let recencySql: string;
|
||||
if (recencyBias === 'on') {
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
recencySql = buildRecencyComponentSql({
|
||||
slugColumn: 'p.slug',
|
||||
dateExpr: 'COALESCE(p.effective_date, p.updated_at)',
|
||||
|
||||
+24
-80
@@ -13,29 +13,7 @@ import type {
|
||||
NewFact, FactListOpts, FactsHealth,
|
||||
SourceRow,
|
||||
} from './engine.ts';
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import { isConnectionEndedError } from './retry-matcher.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { logDbDisconnect } from './audit/db-disconnect-audit.ts';
|
||||
import { logPoolRecovery } from './audit/pool-recovery-audit.ts';
|
||||
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
|
||||
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
|
||||
import type {
|
||||
DomainBankSampleOpts, CorpusSampleOpts, DomainBankRow,
|
||||
@@ -353,6 +331,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// even a no-op disconnect (engine that was never connected) is
|
||||
// recorded — that case may itself be a caller-side bug worth seeing.
|
||||
try {
|
||||
const { logDbDisconnect } = await import('./audit/db-disconnect-audit.ts');
|
||||
logDbDisconnect('postgres', this._connectionStyle ?? 'unknown');
|
||||
} catch { /* best-effort; never block disconnect on audit failure */ }
|
||||
// v0.30.1: tear down the direct pool first if the manager owns one.
|
||||
@@ -402,13 +381,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
let dims: number = DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
|
||||
const sqlText = getPostgresSchema(dims, model);
|
||||
@@ -1056,8 +1031,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
const rows = await tx`
|
||||
SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
FROM pages
|
||||
WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
|
||||
LIMIT 1
|
||||
@@ -2404,8 +2378,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
if (err instanceof Error && err.name === 'RetryAbortError') throw err;
|
||||
// Best-effort exhausted-retry log. If the error wasn't retryable in
|
||||
// the first place, isRetryableConnError(err) is false and we skip.
|
||||
// retry.ts is already in this module's static graph through withRetry, so
|
||||
// classifying the exhausted error does not need a second runtime import.
|
||||
// Lazy-import to avoid a circular dep concern.
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
if (isRetryableConnError(err)) {
|
||||
auditLogBatchExhausted(auditSite, batchSize, opts.maxRetries + 1, err);
|
||||
}
|
||||
@@ -2463,30 +2437,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
// hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors
|
||||
// were produced by a different, config-resolved model — corrupting the
|
||||
// provenance that signature-drift staleness + dim-migration logic trust.
|
||||
//
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured —
|
||||
// it never returns falsy — so an `||` guard here is dead code and the
|
||||
// catch path used to stamp the compile-time default onto rows whose
|
||||
// vectors came from the config-resolved provider. On the throw path we
|
||||
// now fall back to the brain's own `config.embedding_model` row (kept
|
||||
// current by init / migrate / retrieval-upgrade), which names the model
|
||||
// that actually produced this brain's vectors. The compile-time default
|
||||
// is the LAST resort (fresh brain whose config row doesn't exist yet).
|
||||
let resolvedModel: string | null = null;
|
||||
// Mirrors the resolve-then-fallback pattern used for schema sizing above.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
} catch {
|
||||
try {
|
||||
const cfg = await sql`SELECT value FROM config WHERE key = 'embedding_model'`;
|
||||
resolvedModel = (cfg[0]?.value as string | undefined) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2550,11 +2508,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch — the label must
|
||||
// describe whichever vector WINS the upsert. The old COALESCE(EXCLUDED.model, …)
|
||||
// relabeled preserved (older-model) vectors with the current gateway model on every
|
||||
// partial re-embed, corrupting provenance without changing the vector.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2568,14 +2521,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
@@ -4008,6 +3954,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
async mergeOntologyFact(obs: OntologyObservationInput): Promise<OntologyMergeResult> {
|
||||
const sql = this.sql;
|
||||
const sourceId = obs.sourceId ?? 'default';
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const dimension = normalizeDimension(obs.dimension);
|
||||
const vh = valueHash(obs.value);
|
||||
const conf = obs.confidence ?? 0.7;
|
||||
@@ -5456,16 +5403,12 @@ export class PostgresEngine implements BrainEngine {
|
||||
// no outbound links). The raw islanded list is filtered through the same
|
||||
// policy as `gbrain orphans` so convention pages do not count against
|
||||
// dashboard health.
|
||||
// #1305: every page-scoped count here excludes soft-deleted rows — same
|
||||
// posture as getStats — so brain_score moves when the user deletes pages.
|
||||
// Chunk/link counts stay raw (storage until the purge phase), matching
|
||||
// getStats, and destructive-removal counts elsewhere deliberately stay raw.
|
||||
const [h] = await sql`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
@@ -5487,7 +5430,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL
|
||||
WHERE p.type IN ('entity', 'person', 'company')
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`;
|
||||
@@ -5506,7 +5449,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded,
|
||||
EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline
|
||||
FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const pageCount = Number(h.page_count);
|
||||
@@ -5598,17 +5540,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Sync
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number> {
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void> {
|
||||
newSlug = validateSlug(newSlug);
|
||||
const sql = this.sql;
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// Source-qualify so a rename in source A doesn't sweep up same-slug rows
|
||||
// in sources B/C/D (which would either rename them all OR fail the
|
||||
// (source_id, slug) UNIQUE if the new slug already exists in another source).
|
||||
const result = await sql`UPDATE pages SET slug = ${newSlug}, updated_at = now() WHERE slug = ${oldSlug} AND source_id = ${sourceId}`;
|
||||
// #3056: rows moved — a zero-row UPDATE does not throw, so the count is
|
||||
// the only way callers can see the no-op.
|
||||
return result.count ?? 0;
|
||||
await sql`UPDATE pages SET slug = ${newSlug}, updated_at = now() WHERE slug = ${oldSlug} AND source_id = ${sourceId}`;
|
||||
}
|
||||
|
||||
async rewriteLinks(_oldSlug: string, _newSlug: string): Promise<void> {
|
||||
@@ -5847,10 +5786,12 @@ export class PostgresEngine implements BrainEngine {
|
||||
let isReap = false;
|
||||
if (ctx?.error !== undefined) {
|
||||
try {
|
||||
const { isConnectionEndedError } = await import('./retry-matcher.ts');
|
||||
isReap = isConnectionEndedError(ctx.error);
|
||||
} catch { /* classification is best-effort */ }
|
||||
}
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery(isReap ? 'reap_detected' : 'reconnect_other', ctx?.error);
|
||||
} catch { /* audit is best-effort */ }
|
||||
|
||||
@@ -5874,6 +5815,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// New pool is live — discard the old one best-effort.
|
||||
if (oldSql) { try { await oldSql.end({ timeout: 5 }); } catch { /* swallow */ } }
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery('reconnect_succeeded');
|
||||
} catch { /* best-effort */ }
|
||||
} catch (err) {
|
||||
@@ -5885,6 +5827,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
this._sql = oldSql;
|
||||
this.connectionManager = oldManager;
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery('reconnect_failed', err);
|
||||
} catch { /* best-effort */ }
|
||||
throw err; // let batchRetry's backoff handle the retry
|
||||
@@ -6321,6 +6264,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
const recencyBias = opts.recency_bias ?? 'flat';
|
||||
let recencySql: string;
|
||||
if (recencyBias === 'on') {
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
recencySql = buildRecencyComponentSql({
|
||||
slugColumn: 'p.slug',
|
||||
dateExpr: 'COALESCE(p.effective_date, p.updated_at)',
|
||||
|
||||
@@ -48,32 +48,6 @@ import {
|
||||
|
||||
export const RRF_K = 60;
|
||||
const COMPILED_TRUTH_BOOST = 2.0;
|
||||
|
||||
/**
|
||||
* Which detail levels get the compiled_truth boost (#3430).
|
||||
*
|
||||
* ONLY `low`. The documented contract (`src/core/operations.ts`) is
|
||||
* "low (compiled truth only), medium (default, all with dedup), high (all
|
||||
* chunks)" — so `low` is the level that privileges compiled truth, and both
|
||||
* `medium` and `high` are supposed to see everything on equal footing.
|
||||
*
|
||||
* This was previously spelled `detail !== 'high'`, i.e. written as though
|
||||
* `high` were the special case. Because COMPILED_TRUTH_BOOST is applied AFTER
|
||||
* RRF normalization, and RRF's whole range over a 100-deep pool is 1/60 → 1/160,
|
||||
* a 2.0x multiplier is not a tilt — break-even is `2/(60+r) >= 1/60`, so any
|
||||
* boosted chunk inside the first 60 ranks outranks an unboosted rank-1 chunk.
|
||||
* At the default detail that made search categorically compiled-truth-only:
|
||||
* a page whose answer lived in a `fenced_code` chunk returned the prose chunk,
|
||||
* and the code chunk fell out of the window entirely.
|
||||
*
|
||||
* Extracted as a named predicate rather than left inline at three call sites so
|
||||
* the detail→boost mapping is directly testable. An inline expression can only
|
||||
* be covered through a full `hybridSearch` round trip, which is why the
|
||||
* original inversion went unnoticed.
|
||||
*/
|
||||
export function shouldBoostCompiledTruth(detail: string | null | undefined): boolean {
|
||||
return detail === 'low';
|
||||
}
|
||||
const pendingCacheWrites = new Set<Promise<unknown>>();
|
||||
|
||||
/**
|
||||
@@ -1195,7 +1169,7 @@ export async function hybridSearch(
|
||||
const noEmbedLists = [{ list: keywordResults, k: fk }];
|
||||
if (titleResults.length > 0) noEmbedLists.push({ list: titleResults, k: fk });
|
||||
if (relationalList.length > 0) noEmbedLists.push({ list: relationalList, k: fk });
|
||||
noEmbedResults = rrfFusionWeighted(noEmbedLists, shouldBoostCompiledTruth(detailResolved));
|
||||
noEmbedResults = rrfFusionWeighted(noEmbedLists, detailResolved !== 'high');
|
||||
}
|
||||
if (noEmbedResults.length > 0) {
|
||||
await runPostFusionStages(engine, noEmbedResults, postFusionOpts);
|
||||
@@ -1439,7 +1413,7 @@ export async function hybridSearch(
|
||||
const fallbackLists = [{ list: keywordResults, k: fk }];
|
||||
if (titleResults.length > 0) fallbackLists.push({ list: titleResults, k: fk });
|
||||
if (relationalList.length > 0) fallbackLists.push({ list: relationalList, k: fk });
|
||||
fallbackResults = rrfFusionWeighted(fallbackLists, shouldBoostCompiledTruth(detail));
|
||||
fallbackResults = rrfFusionWeighted(fallbackLists, detail !== 'high');
|
||||
}
|
||||
if (fallbackResults.length > 0) {
|
||||
await runPostFusionStages(engine, fallbackResults, postFusionOpts);
|
||||
@@ -1526,7 +1500,7 @@ export async function hybridSearch(
|
||||
// arms BEFORE fusion so the compiled-truth authority boost skips them.
|
||||
await stampUnverifiedExtractions(engine, allLists.flatMap((l) => l.list));
|
||||
|
||||
let fused = rrfFusionWeighted(allLists, shouldBoostCompiledTruth(detail));
|
||||
let fused = rrfFusionWeighted(allLists, detail !== 'high');
|
||||
|
||||
// Cosine re-scoring before dedup so semantically better chunks survive.
|
||||
// v0.36 (D9): hydrate from the active embedding column so rescore happens
|
||||
|
||||
@@ -766,7 +766,7 @@ export function attributeKnob<K extends keyof ModeBundle>(
|
||||
// written between the #3391 stale-fix (which changes which chunks count as
|
||||
// current) and the operator's migration run. Same one-time global cold-miss
|
||||
// pattern as the bumps above.
|
||||
export const KNOBS_HASH_VERSION = 14;
|
||||
export const KNOBS_HASH_VERSION = 13;
|
||||
|
||||
/**
|
||||
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
|
||||
|
||||
@@ -33,17 +33,6 @@
|
||||
|
||||
export const SOURCE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$/;
|
||||
|
||||
/**
|
||||
* Sentinel meaning "span every source" (#1712). Deliberately NOT a valid
|
||||
* source id (underscores are rejected by SOURCE_ID_RE), so it can never
|
||||
* collide with a real source, be created via `sources add`, or leak into
|
||||
* lock ids / path joins. The resolver's explicit/env tiers pass it through
|
||||
* verbatim; `sourceScopeOpts` translates it to an unscoped read for trusted
|
||||
* local callers and keeps it as an unsatisfiable literal for remote callers
|
||||
* (fail-closed).
|
||||
*/
|
||||
export const ALL_SOURCES = '__all__';
|
||||
|
||||
/** Returns true if the string matches the canonical source_id regex. */
|
||||
export function isValidSourceId(s: unknown): s is string {
|
||||
return typeof s === 'string' && SOURCE_ID_RE.test(s);
|
||||
|
||||
@@ -17,13 +17,9 @@ import { readFileSync, lstatSync, type Stats } from 'fs';
|
||||
import { join, dirname, resolve } from 'path';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { isSourceFederated } from './sources-load.ts';
|
||||
import { SOURCE_ID_RE, isValidSourceId, ALL_SOURCES } from './source-id.ts';
|
||||
import { SOURCE_ID_RE, isValidSourceId } from './source-id.ts';
|
||||
import { isTrustedDotfile, realpathOrResolve } from './path-confine.ts';
|
||||
|
||||
// Re-export so scope-resolution call sites can import the sentinel from
|
||||
// either module (#1712).
|
||||
export { ALL_SOURCES };
|
||||
|
||||
const DOTFILE = '.gbrain-source';
|
||||
// Canonical SOURCE_ID_RE imported from `source-id.ts` (single source of truth).
|
||||
// Re-exported below as `__testing.SOURCE_ID_RE` for legacy test imports.
|
||||
@@ -87,11 +83,8 @@ export async function resolveSourceId(
|
||||
explicit: string | null | undefined,
|
||||
cwd: string = process.cwd(),
|
||||
): Promise<string> {
|
||||
// 1. Explicit flag wins. The __all__ sentinel passes through verbatim
|
||||
// (#1712) — it is not a source id, so it skips both the regex and
|
||||
// assertSourceExists; sourceScopeOpts gives it span-everything semantics.
|
||||
// 1. Explicit flag wins.
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) return ALL_SOURCES;
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -99,10 +92,9 @@ export async function resolveSourceId(
|
||||
return explicit;
|
||||
}
|
||||
|
||||
// 2. Env var. Same __all__ pass-through (#2140).
|
||||
// 2. Env var.
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) return ALL_SOURCES;
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -181,7 +173,6 @@ export function resolveSourceIdEngineFree(
|
||||
cwd: string = process.cwd(),
|
||||
): string | null {
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) return ALL_SOURCES; // #1712 sentinel pass-through
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -189,7 +180,6 @@ export function resolveSourceIdEngineFree(
|
||||
}
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) return ALL_SOURCES; // #2140 sentinel pass-through
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -325,11 +315,8 @@ export async function resolveSourceWithTier(
|
||||
explicit: string | null | undefined,
|
||||
cwd: string = process.cwd(),
|
||||
): Promise<{ source_id: string; tier: SourceTier; detail?: string }> {
|
||||
// 1. Explicit flag wins. __all__ sentinel passes through verbatim (#1712).
|
||||
// 1. Explicit flag wins.
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) {
|
||||
return { source_id: ALL_SOURCES, tier: 'flag', detail: `--source ${ALL_SOURCES} (spans all sources)` };
|
||||
}
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -337,12 +324,9 @@ export async function resolveSourceWithTier(
|
||||
return { source_id: explicit, tier: 'flag', detail: `--source ${explicit}` };
|
||||
}
|
||||
|
||||
// 2. Env var. Same __all__ pass-through (#2140).
|
||||
// 2. Env var.
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) {
|
||||
return { source_id: ALL_SOURCES, tier: 'env', detail: `GBRAIN_SOURCE=${ALL_SOURCES} (spans all sources)` };
|
||||
}
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
|
||||
+8
-13
@@ -11,7 +11,7 @@
|
||||
* pathToSlug() → convert file paths to page slugs
|
||||
*/
|
||||
|
||||
import { SLUG_WORD_CHARS } from './cjk.ts';
|
||||
import { CJK_SLUG_CHARS } from './cjk.ts';
|
||||
// v0.37.7.0 #1169 submodule-detection helpers. Bottom-of-file already
|
||||
// aliases existsSync as `_existsSync` for other purposes; the top-of-file
|
||||
// import keeps the pruneDir helper's deps near its callsite.
|
||||
@@ -396,10 +396,8 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync
|
||||
|
||||
/**
|
||||
* Character class for the lowercase-canonical form of a slug segment after
|
||||
* slugifySegment() has run. Letters/numbers in any script (lowercase where
|
||||
* the script has case — #3417), dots, underscores, hyphens. Uses \p{...}
|
||||
* classes, so composed regexes need the `u` flag (this one carries it).
|
||||
* Exposed so adjacent code (e.g. takes-fence holder validation,
|
||||
* slugifySegment() has run. Lowercase letters, digits, dots, underscores,
|
||||
* hyphens. Exposed so adjacent code (e.g. takes-fence holder validation,
|
||||
* v0.32 EXP-4) can reuse the actual repo slug grammar instead of inventing
|
||||
* a stricter parallel one and emitting false-positive warnings on legitimate
|
||||
* `companies/acme.io` / `people/foo_bar` slugs (codex review #3).
|
||||
@@ -407,18 +405,15 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync
|
||||
* Pattern is the inner character class only (no anchors); callers wrap it
|
||||
* in `^...$` or compose it with prefixes like `(?:people|companies)/...`.
|
||||
*/
|
||||
export const SLUG_SEGMENT_PATTERN = new RegExp(`[${SLUG_WORD_CHARS}._\\-]+`, 'u');
|
||||
export const SLUG_SEGMENT_PATTERN = new RegExp(`[a-z0-9._\\-${CJK_SLUG_CHARS}]+`);
|
||||
|
||||
/**
|
||||
* Slugify a single path segment: lowercase, strip special chars, spaces → hyphens.
|
||||
* Letters and numbers from EVERY script are preserved (#3417): previously only
|
||||
* Latin + CJK survived, so Hebrew/Arabic/Cyrillic/Greek/Thai/... filenames
|
||||
* collapsed to empty segments and distinct files silently merged onto one slug.
|
||||
* NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes
|
||||
* back into precomposed syllables, and so NFD filenames (macOS) and NFC
|
||||
* filenames (Linux/git) of the same name produce the SAME slug.
|
||||
* CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) are preserved (v0.32.7).
|
||||
* NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes back
|
||||
* into precomposed syllables that fall inside the whitelist.
|
||||
*/
|
||||
const SLUGIFY_KEEP_RE = new RegExp(`[^${SLUG_WORD_CHARS}.\\s_\\-]`, 'gu');
|
||||
const SLUGIFY_KEEP_RE = new RegExp(`[^a-z0-9.\\s_\\-${CJK_SLUG_CHARS}]`, 'g');
|
||||
|
||||
export function slugifySegment(segment: string): string {
|
||||
return segment
|
||||
|
||||
@@ -134,7 +134,6 @@ export const TAKES_FENCE_END = '<!--- gbrain:takes:end -->';
|
||||
import { SLUG_SEGMENT_PATTERN } from './sync.ts';
|
||||
export const HOLDER_REGEX = new RegExp(
|
||||
`^(?:world|brain|(?:people|companies)/${SLUG_SEGMENT_PATTERN.source}|${SLUG_SEGMENT_PATTERN.source})$`,
|
||||
'u', // required by SLUG_SEGMENT_PATTERN's \p{...} classes (#3417)
|
||||
);
|
||||
|
||||
/**
|
||||
|
||||
@@ -110,12 +110,6 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
const sourceUri = row.source_uri === undefined ? undefined : (row.source_uri as string | null);
|
||||
const ingestedVia = row.ingested_via === undefined ? undefined : (row.ingested_via as string | null);
|
||||
const ingestedAt = readOptionalDate(row.ingested_at);
|
||||
// #3507: the CR tier the page was last embedded under (three-state, same
|
||||
// pattern as the provenance columns above). Re-embed paths (`embed --stale`
|
||||
// and friends) read this to reproduce the page's stored wrapping convention.
|
||||
const contextualRetrievalMode = row.contextual_retrieval_mode === undefined
|
||||
? undefined
|
||||
: (row.contextual_retrieval_mode as Page['contextual_retrieval_mode']);
|
||||
return {
|
||||
id: row.id as number,
|
||||
slug: row.slug as string,
|
||||
@@ -141,7 +135,6 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
...(sourceUri !== undefined && { source_uri: sourceUri }),
|
||||
...(ingestedVia !== undefined && { ingested_via: ingestedVia }),
|
||||
...(ingestedAt !== undefined && { ingested_at: ingestedAt }),
|
||||
...(contextualRetrievalMode !== undefined && { contextual_retrieval_mode: contextualRetrievalMode }),
|
||||
// v0.31.12: propagate source_id so downstream callers (embed, reconcile-links)
|
||||
// can thread it through getChunks / upsertChunks without defaulting to 'default'.
|
||||
// v0.32.8: Page.source_id is required. Every SELECT feeding rowToPage now
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { openAdminSseStream, type AdminSseResponse } from '../src/commands/serve-http.ts';
|
||||
|
||||
describe('admin SSE handshake', () => {
|
||||
test('flushes a protocol-valid comment immediately after the headers', () => {
|
||||
const calls: string[] = [];
|
||||
const headers = new Map<string, string>();
|
||||
|
||||
openAdminSseStream({
|
||||
setHeader(name: string, value: string | number | readonly string[]) {
|
||||
headers.set(name, String(value));
|
||||
calls.push(`header:${name}`);
|
||||
return this;
|
||||
},
|
||||
flushHeaders() {
|
||||
calls.push('flush');
|
||||
},
|
||||
write(chunk: unknown) {
|
||||
calls.push(`write:${String(chunk)}`);
|
||||
return true;
|
||||
},
|
||||
} as unknown as AdminSseResponse);
|
||||
|
||||
expect(headers).toEqual(new Map([
|
||||
['Content-Type', 'text/event-stream'],
|
||||
['Cache-Control', 'no-cache'],
|
||||
['Connection', 'keep-alive'],
|
||||
]));
|
||||
expect(calls).toEqual([
|
||||
'header:Content-Type',
|
||||
'header:Cache-Control',
|
||||
'header:Connection',
|
||||
'flush',
|
||||
'write:: connected\n\n',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,68 +0,0 @@
|
||||
/**
|
||||
* #3554 — resetGateway() must restore the test baseline, not unconfigure.
|
||||
*
|
||||
* The bunfig preload (test/helpers/legacy-embedding-preload.ts) pins the
|
||||
* gateway to openai:text-embedding-3-large @ 1536 at process start and
|
||||
* registers that config as the reset baseline. Before the fix,
|
||||
* resetGateway() wiped the pin to _config = null; the next file's beforeAll
|
||||
* engine-connect then reconfigured from the SHIPPED default (zembed-1 @
|
||||
* 1280) and every 1536-d fixture in that file failed with
|
||||
* `expected 1280 dimensions, not 1536`. Which file pairs collided depended
|
||||
* on shard bin-packing, so adding ANY test file reshuffled the mines.
|
||||
*
|
||||
* These assertions pin the contract so it cannot silently rot again.
|
||||
*/
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__unconfigureGatewayForTests,
|
||||
__setChatTransportForTests,
|
||||
getEmbeddingModel,
|
||||
getEmbeddingDimensions,
|
||||
isAvailable,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
|
||||
afterEach(() => resetGateway());
|
||||
|
||||
describe('resetGateway baseline restore (#3554)', () => {
|
||||
test('immediately after resetGateway(), the preload baseline is live', () => {
|
||||
resetGateway();
|
||||
expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large');
|
||||
expect(getEmbeddingDimensions()).toBe(1536);
|
||||
});
|
||||
|
||||
test('resetGateway() overwrites a file-local config back to the baseline', () => {
|
||||
configureGateway({
|
||||
embedding_model: 'zeroentropyai:zembed-1',
|
||||
embedding_dimensions: 1280,
|
||||
env: {},
|
||||
});
|
||||
expect(getEmbeddingDimensions()).toBe(1280);
|
||||
resetGateway();
|
||||
expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large');
|
||||
expect(getEmbeddingDimensions()).toBe(1536);
|
||||
});
|
||||
|
||||
test('resetGateway() still clears test transports (no stale transport leaks back)', () => {
|
||||
__setChatTransportForTests(async () => {
|
||||
throw new Error('should have been cleared');
|
||||
});
|
||||
resetGateway();
|
||||
// Baseline config sets no chat key in a keyless env, but the transport
|
||||
// seam itself must be gone: isAvailable('chat') short-circuits to true
|
||||
// whenever a chat transport is installed, so with a hard-unconfigured
|
||||
// gateway it can only be true if the transport survived the reset.
|
||||
__unconfigureGatewayForTests();
|
||||
expect(isAvailable('chat')).toBe(false);
|
||||
});
|
||||
|
||||
test('__unconfigureGatewayForTests() gives a genuinely unconfigured gateway', () => {
|
||||
__unconfigureGatewayForTests();
|
||||
expect(() => getEmbeddingDimensions()).toThrow(/not configured/);
|
||||
expect(isAvailable('embedding')).toBe(false);
|
||||
// And a plain reset brings the baseline back.
|
||||
resetGateway();
|
||||
expect(getEmbeddingDimensions()).toBe(1536);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,6 @@ import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__unconfigureGatewayForTests,
|
||||
isAvailable,
|
||||
embed,
|
||||
getEmbeddingModel,
|
||||
@@ -56,9 +55,6 @@ describe('gateway.isAvailable (silent-drop regression surface)', () => {
|
||||
beforeEach(() => resetGateway());
|
||||
|
||||
test('returns false when gateway not configured', () => {
|
||||
// resetGateway() restores the preload's test baseline (#3554); go
|
||||
// genuinely unconfigured for this one assertion.
|
||||
__unconfigureGatewayForTests();
|
||||
expect(isAvailable('embedding')).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
/**
|
||||
* #1712 (dupes #2289, #2140) — the `__all__` sentinel must work in EVERY
|
||||
* resolution tier, not just as a per-call `source_id` param.
|
||||
*
|
||||
* The bug: SOURCE_ID_RE forbids underscores, so `--source __all__` and
|
||||
* `GBRAIN_SOURCE=__all__` threw in the resolver; the CLI's makeContext
|
||||
* blanket-caught that and silently fell back to `sourceId: 'default'` —
|
||||
* making the documented span-everything flag STRICTLY NARROWER than passing
|
||||
* no flag at all (the catch also discarded the #2561/#3242 federated
|
||||
* widening). Meanwhile sourceScopeOpts treated a ctx.sourceId of '__all__'
|
||||
* as an unsatisfiable literal.
|
||||
*
|
||||
* Uses the literal '__all__' (not the ALL_SOURCES constant) so these tests
|
||||
* load and run behaviorally against pre-fix trees.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import {
|
||||
resolveSourceId,
|
||||
resolveSourceIdEngineFree,
|
||||
resolveSourceWithTier,
|
||||
} from '../src/core/source-resolver.ts';
|
||||
import {
|
||||
sourceScopeOpts,
|
||||
federatedSearchScope,
|
||||
type OperationContext,
|
||||
} from '../src/core/operations.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
// Stub engine: registered sources + no local_path rows + no default config.
|
||||
function makeStub(registeredSources: string[]): BrainEngine {
|
||||
return {
|
||||
kind: 'pglite',
|
||||
executeRaw: async <T>(sql: string, params?: unknown[]): Promise<T[]> => {
|
||||
if (sql.includes('SELECT id FROM sources WHERE id = $1')) {
|
||||
const target = params?.[0];
|
||||
return registeredSources.includes(target as string)
|
||||
? [{ id: target } as unknown as T]
|
||||
: [];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
getConfig: async () => null,
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
function ctxOf(overrides: Partial<OperationContext> = {}): OperationContext {
|
||||
return {
|
||||
engine: {} as any,
|
||||
config: {} as any,
|
||||
logger: console as any,
|
||||
dryRun: false,
|
||||
remote: true,
|
||||
sourceId: 'default',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Resolver tiers pass the sentinel through verbatim ──────────────────
|
||||
|
||||
describe('source-resolver — __all__ sentinel pass-through', () => {
|
||||
test('resolveSourceId: explicit --source __all__ resolves (no regex throw, no existence check)', async () => {
|
||||
// '__all__' is deliberately NOT in the registered set — the sentinel
|
||||
// must skip assertSourceExists (it is not a source id).
|
||||
const id = await resolveSourceId(makeStub(['default']), '__all__', '/nonexistent');
|
||||
expect(id).toBe('__all__');
|
||||
});
|
||||
|
||||
test('resolveSourceId: GBRAIN_SOURCE=__all__ resolves (#2140)', async () => {
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, async () => {
|
||||
const id = await resolveSourceId(makeStub(['default']), null, '/nonexistent');
|
||||
expect(id).toBe('__all__');
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveSourceIdEngineFree: explicit + env __all__ (thin-client path)', async () => {
|
||||
expect(resolveSourceIdEngineFree('__all__', '/nonexistent')).toBe('__all__');
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, () => {
|
||||
expect(resolveSourceIdEngineFree(null, '/nonexistent')).toBe('__all__');
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveSourceWithTier: flag and env tiers carry the sentinel', async () => {
|
||||
const flag = await resolveSourceWithTier(makeStub(['default']), '__all__', '/nonexistent');
|
||||
expect(flag).toMatchObject({ source_id: '__all__', tier: 'flag' });
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, async () => {
|
||||
const env = await resolveSourceWithTier(makeStub(['default']), null, '/nonexistent');
|
||||
expect(env).toMatchObject({ source_id: '__all__', tier: 'env' });
|
||||
});
|
||||
});
|
||||
|
||||
test('a genuinely invalid --source still throws (SOURCE_ID_RE not loosened)', async () => {
|
||||
await expect(resolveSourceId(makeStub(['default']), 'my_source', '/nonexistent'))
|
||||
.rejects.toThrow(/Invalid --source/);
|
||||
expect(() => resolveSourceIdEngineFree('my_source', '/nonexistent'))
|
||||
.toThrow(/Invalid --source/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── sourceScopeOpts — the single read-scope choke point ─────────────────
|
||||
|
||||
describe('sourceScopeOpts — __all__ sentinel', () => {
|
||||
test('trusted local (remote === false): spans the whole brain (empty scope)', () => {
|
||||
expect(sourceScopeOpts(ctxOf({ remote: false, sourceId: '__all__' }))).toEqual({});
|
||||
});
|
||||
|
||||
test('remote: keeps the unsatisfiable literal — fail-closed, never widens', () => {
|
||||
expect(sourceScopeOpts(ctxOf({ remote: true, sourceId: '__all__' })))
|
||||
.toEqual({ sourceId: '__all__' });
|
||||
});
|
||||
|
||||
test('anything not strictly remote === false is untrusted (fail-closed)', () => {
|
||||
// undefined / missing remote must behave like remote, per the trust rule.
|
||||
const ctx = ctxOf({ sourceId: '__all__' });
|
||||
(ctx as any).remote = undefined;
|
||||
expect(sourceScopeOpts(ctx)).toEqual({ sourceId: '__all__' });
|
||||
});
|
||||
|
||||
test('a federated grant always wins over the sentinel', () => {
|
||||
const ctx = ctxOf({
|
||||
remote: true,
|
||||
sourceId: '__all__',
|
||||
auth: { token: 't', clientId: 'c', scopes: [], allowedSources: ['a', 'b'] } as any,
|
||||
});
|
||||
expect(sourceScopeOpts(ctx)).toEqual({ sourceIds: ['a', 'b'] });
|
||||
});
|
||||
});
|
||||
|
||||
// ── Never narrower than passing no flag (#2561 regression shape) ────────
|
||||
|
||||
describe('__all__ is never narrower than an unqualified read', () => {
|
||||
test('local __all__ spans the brain even when federated widening exists', () => {
|
||||
// Unqualified read on a federated brain widens to the federated array…
|
||||
const unqualified = ctxOf({
|
||||
remote: false,
|
||||
sourceId: 'default',
|
||||
localFederatedSourceIds: ['default', 'src-a', 'src-b'],
|
||||
});
|
||||
expect(federatedSearchScope(unqualified)).toEqual({
|
||||
sourceIds: ['default', 'src-a', 'src-b'],
|
||||
});
|
||||
// …and __all__ must be a superset of that: the whole brain ({}).
|
||||
const all = ctxOf({ remote: false, sourceId: '__all__' });
|
||||
expect(federatedSearchScope(all)).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeContext — explicit --source failures error loudly ───────────────
|
||||
|
||||
describe('cli makeContext — no silent default fallback for explicit --source', () => {
|
||||
test('--source __all__ produces ctx.sourceId __all__ (was: silent default)', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
const ctx = await makeContext(makeStub(['default']), { source: '__all__' });
|
||||
expect(ctx.sourceId).toBe('__all__');
|
||||
expect(ctx.remote).toBe(false);
|
||||
});
|
||||
|
||||
test('an explicit --source that fails to resolve throws instead of becoming default', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
await expect(makeContext(makeStub(['default']), { source: 'ghost' }))
|
||||
.rejects.toThrow(/not found/);
|
||||
await expect(makeContext(makeStub(['default']), { source: 'my_source' }))
|
||||
.rejects.toThrow(/Invalid --source/);
|
||||
});
|
||||
|
||||
test('ambient resolution failure still falls back silently (pre-init brains)', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
const broken = {
|
||||
kind: 'pglite',
|
||||
executeRaw: async () => { throw new Error('relation "sources" does not exist'); },
|
||||
getConfig: async () => { throw new Error('relation "config" does not exist'); },
|
||||
} as unknown as BrainEngine;
|
||||
const ctx = await makeContext(broken, {});
|
||||
expect(ctx.sourceId).toBe('default');
|
||||
});
|
||||
});
|
||||
@@ -11,34 +11,15 @@ import { tmpdir } from 'os';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { hardenBrainRepo } from '../src/core/brain-repo-durability.ts';
|
||||
|
||||
// #2943 root cause: `env: process.env` is REQUIRED here. Bun snapshots
|
||||
// process.env at startup, so without it the spawned git — and any post-commit
|
||||
// hook it fires — is blind to beforeEach's HOME/GBRAIN_HOME mutations (the
|
||||
// same Bun quirk as #2747, see resolveGbrainCliPath in brain-repo-durability).
|
||||
// Pre-fix, the hook under test resolved ${GBRAIN_HOME:-$HOME/.gbrain} to the
|
||||
// OPERATOR'S REAL ~/.gbrain: it wrote its log lines there (polluting the real
|
||||
// brain-push.log on every run), the LOCAL-ONLY test never saw them in the
|
||||
// temp log it polls, and the assertion only passed when the scaffolding push
|
||||
// from beforeEach (spawned by hardenBrainRepo WITH explicit env) happened to
|
||||
// still be in flight, lose the ref race, and retry AFTER the test had pointed
|
||||
// origin at the dead path — an accidental, load-dependent signal. That race
|
||||
// is the CI flake.
|
||||
function git(cwd: string, ...args: string[]): string {
|
||||
return execFileSync('git', ['-C', cwd, '-c', 'protocol.file.allow=always', ...args], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', env: process.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8',
|
||||
}).trim();
|
||||
}
|
||||
function originHead(bare: string): string {
|
||||
return git(bare, 'rev-parse', 'refs/heads/main');
|
||||
}
|
||||
// #2943: 30s poll deadlines (was 8s) for headroom under loaded CI shards —
|
||||
// the unreachable-origin path runs ~6 sequential process spawns after the
|
||||
// hook detaches. Every hook test also passes an explicit 60_000 third-arg
|
||||
// timeout: bun 1.3.14 IGNORES bunfig.toml's `timeout` key, so a bare
|
||||
// `bun test` enforces its 5000ms default and killed these tests before the
|
||||
// internal deadline could even elapse (the runner scripts pass --timeout
|
||||
// explicitly, which is why the inversion only bit direct local runs).
|
||||
async function waitForOrigin(bare: string, expectSha: string, ms = 30_000): Promise<boolean> {
|
||||
async function waitForOrigin(bare: string, expectSha: string, ms = 8000): Promise<boolean> {
|
||||
const deadline = Date.now() + ms;
|
||||
while (Date.now() < deadline) {
|
||||
try { if (originHead(bare) === expectSha) return true; } catch { /* */ }
|
||||
@@ -47,24 +28,6 @@ async function waitForOrigin(bare: string, expectSha: string, ms = 30_000): Prom
|
||||
return false;
|
||||
}
|
||||
|
||||
/** #2943 (index.lock form): hardenBrainRepo installs the post-commit hook
|
||||
* BEFORE committing the scaffolding, so that commit fires the hook and
|
||||
* detaches a background brain_push. If that push loses the ref race against
|
||||
* hardenBrainRepo's own synchronous push, it falls back to `git pull
|
||||
* --rebase`, which takes .git/index.lock — racing the test body's first git
|
||||
* calls ("Unable to create '.../.git/index.lock': File exists"). Wait for the
|
||||
* detached push's terminal log line before handing the repo to the test. */
|
||||
async function waitForHookPushSettled(ms = 30_000): Promise<void> {
|
||||
const log = join(process.env.GBRAIN_HOME!, 'brain-push.log');
|
||||
const terminal = /\[push\] (ok|lock-timeout|LOCAL-ONLY)/;
|
||||
const deadline = Date.now() + ms;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(log) && terminal.test(readFileSync(log, 'utf-8'))) return;
|
||||
await new Promise(r => setTimeout(r, 150));
|
||||
}
|
||||
throw new Error(`detached hook push did not settle within ${ms}ms (${log})`);
|
||||
}
|
||||
|
||||
let root: string, work: string, bare: string;
|
||||
let oldHome: string | undefined, oldGbrainHome: string | undefined;
|
||||
|
||||
@@ -75,15 +38,14 @@ beforeEach(async () => {
|
||||
process.env.GBRAIN_HOME = join(process.env.HOME, '.gbrain');
|
||||
process.env.GBRAIN_GIT_ALLOW_FILE_TRANSPORT = '1';
|
||||
bare = mkdtempSync(join(root, 'origin-')) + '.git';
|
||||
execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare], { stdio: 'ignore', env: process.env });
|
||||
execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare], { stdio: 'ignore' });
|
||||
work = mkdtempSync(join(root, 'work-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, work], { stdio: 'ignore', env: process.env });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, work], { stdio: 'ignore' });
|
||||
git(work, 'config', 'user.email', 't@t.t'); git(work, 'config', 'user.name', 'tester');
|
||||
writeFileSync(join(work, 'README.md'), 'init\n');
|
||||
git(work, 'add', 'README.md'); git(work, 'commit', '-qm', 'init'); git(work, 'push', '-q', 'origin', 'main');
|
||||
git(work, 'remote', 'set-head', 'origin', 'main');
|
||||
await hardenBrainRepo({ repoPath: work, sourceId: 'wiki', pat: 'ghp_x', installCron: false });
|
||||
await waitForHookPushSettled();
|
||||
});
|
||||
afterEach(() => {
|
||||
if (oldHome === undefined) delete process.env.HOME; else process.env.HOME = oldHome;
|
||||
@@ -103,7 +65,7 @@ describe('brain-commit-push.sh (D13 guarantee)', () => {
|
||||
expect(originHead(bare)).toBe(git(work, 'rev-parse', 'HEAD'));
|
||||
// origin actually has the file
|
||||
const verify = mkdtempSync(join(root, 'verify-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, verify], { stdio: 'ignore', env: process.env });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, verify], { stdio: 'ignore' });
|
||||
expect(existsSync(join(verify, 'people', 'alice.md'))).toBe(true);
|
||||
});
|
||||
|
||||
@@ -140,7 +102,7 @@ describe('brain-commit-push.sh (D13 guarantee)', () => {
|
||||
rmSync(join(work, '.git', 'hooks', 'post-commit'));
|
||||
// Advance the remote from a second clone so a pull is genuinely needed.
|
||||
const other = mkdtempSync(join(root, 'other-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, other], { stdio: 'ignore', env: process.env });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, other], { stdio: 'ignore' });
|
||||
git(other, 'config', 'user.email', 'o@o.o'); git(other, 'config', 'user.name', 'other');
|
||||
writeFileSync(join(other, 'remote.md'), 'from other\n');
|
||||
git(other, 'add', 'remote.md'); git(other, 'commit', '-qm', 'remote change'); git(other, 'push', '-q', 'origin', 'main');
|
||||
@@ -166,26 +128,26 @@ describe('post-commit hook (D9 local, D7 self-contained)', () => {
|
||||
git(work, 'add', 'note.md'); git(work, 'commit', '-qm', 'note'); // fires .git/hooks/post-commit
|
||||
const head = git(work, 'rev-parse', 'HEAD');
|
||||
expect(await waitForOrigin(bare, head)).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
test('the hook works even with the committed helper deleted (self-contained)', async () => {
|
||||
rmSync(join(work, 'scripts', 'brain-commit-push.sh'));
|
||||
git(work, 'add', '-A'); git(work, 'commit', '-qm', 'remove helper');
|
||||
const head = git(work, 'rev-parse', 'HEAD');
|
||||
expect(await waitForOrigin(bare, head)).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
test('logs a clear LOCAL-ONLY line when origin is unreachable', async () => {
|
||||
git(work, 'remote', 'set-url', 'origin', join(root, 'gone2.git'));
|
||||
writeFileSync(join(work, 'orphan.md'), 'o\n');
|
||||
git(work, 'add', 'orphan.md'); git(work, 'commit', '-qm', 'orphan');
|
||||
const log = join(process.env.GBRAIN_HOME!, 'brain-push.log');
|
||||
const deadline = Date.now() + 30_000;
|
||||
const deadline = Date.now() + 8000;
|
||||
let found = false;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(log) && readFileSync(log, 'utf-8').includes('NEEDS ATTENTION')) { found = true; break; }
|
||||
await new Promise(r => setTimeout(r, 150));
|
||||
}
|
||||
expect(found).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
/**
|
||||
* #3513: parseOpArgs' stdin read must never block forever.
|
||||
*
|
||||
* In a non-TTY with no piped input — a CI step, a cron job, an agent
|
||||
* harness that inherits a non-TTY stdin without writing to it — the old
|
||||
* inline `readFileSync(0)` never returned. The fix bounds the read with a
|
||||
* first-byte deadline (pipes/sockets only) and falls through to the
|
||||
* existing required-param usage error on timeout.
|
||||
*
|
||||
* The load-bearing regression test spawns the REAL CLI with a held-open,
|
||||
* never-written pipe: on pre-fix code it hangs until our observation window
|
||||
* kills it; on fixed code it exits 1 with the usage error well inside the
|
||||
* window. The stdin read + required-param check both run BEFORE engine
|
||||
* connect, so no brain/DB is touched.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const REPO = dirname(import.meta.dir);
|
||||
const CLI = join(REPO, 'src', 'cli.ts');
|
||||
|
||||
interface CliRun {
|
||||
exited: boolean;
|
||||
exitCode: number | null;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
/** Narrow Bun's `number | FileSink` stdin union to the pipe sink. */
|
||||
function pipeSink(proc: { stdin: unknown }): { write(d: string): unknown; end(): unknown } {
|
||||
const s = proc.stdin;
|
||||
if (!s || typeof s === 'number') throw new Error('expected a piped stdin sink');
|
||||
return s as { write(d: string): unknown; end(): unknown };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn the CLI with the given stdin wiring. `holdPipeOpen` keeps the write
|
||||
* end of the stdin pipe alive without ever writing — the #3513 repro. The
|
||||
* observation window kills the child if it hasn't exited (pre-fix hang).
|
||||
*/
|
||||
async function runCliWithStdin(
|
||||
args: string[],
|
||||
stdin: 'hold-open' | 'closed-empty' | { data: string } | { file: string },
|
||||
windowMs: number,
|
||||
): Promise<CliRun> {
|
||||
const proc = Bun.spawn(['bun', 'run', CLI, ...args], {
|
||||
cwd: REPO,
|
||||
env: { ...process.env, GBRAIN_STDIN_TIMEOUT_MS: '500' },
|
||||
stdin: typeof stdin === 'object' && 'file' in stdin ? Bun.file(stdin.file) : 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
if (typeof stdin === 'object' && 'data' in stdin) {
|
||||
pipeSink(proc).write(stdin.data);
|
||||
await pipeSink(proc).end();
|
||||
} else if (stdin === 'closed-empty') {
|
||||
await pipeSink(proc).end();
|
||||
}
|
||||
// 'hold-open': never write, never close — the CI/cron/agent-harness shape.
|
||||
|
||||
let exited = true;
|
||||
const killer = setTimeout(() => {
|
||||
exited = false;
|
||||
try { proc.kill('SIGKILL'); } catch { /* already dead */ }
|
||||
}, windowMs);
|
||||
const [exitCode, stderr] = await Promise.all([
|
||||
proc.exited,
|
||||
new Response(proc.stderr).text(),
|
||||
]);
|
||||
clearTimeout(killer);
|
||||
try { pipeSink(proc).end(); } catch { /* hold-open cleanup */ }
|
||||
return { exited, exitCode: exited ? exitCode : null, stderr };
|
||||
}
|
||||
|
||||
describe('#3513 — stdin-capable op with a non-TTY, never-written stdin', () => {
|
||||
test('exits fast with the usage error instead of blocking forever', async () => {
|
||||
// `put` declares stdin:'content' (required). No inline content, no piped
|
||||
// input → the bounded read times out at 500ms, content stays unset, and
|
||||
// the required-param check prints usage and exits 1. Pre-fix: readFileSync(0)
|
||||
// blocks until the 20s window kills the child.
|
||||
const run = await runCliWithStdin(['put', 'stdin-hang-test-slug'], 'hold-open', 20_000);
|
||||
expect(run.exited).toBe(true); // pre-#3513 this is false: the read never returns
|
||||
expect(run.exitCode).toBe(1);
|
||||
expect(run.stderr).toContain('Usage: gbrain put');
|
||||
}, 30_000);
|
||||
|
||||
test('a genuine pipe with data is still consumed (no hang, no crash)', async () => {
|
||||
// Piped content fills `content`; the missing positional slug then fails
|
||||
// the required check — proving the stream path read stdin and moved on.
|
||||
const run = await runCliWithStdin(['put'], { data: '# hello\n' }, 20_000);
|
||||
expect(run.exited).toBe(true);
|
||||
expect(run.exitCode).toBe(1);
|
||||
expect(run.stderr).toContain('Usage: gbrain put');
|
||||
}, 30_000);
|
||||
|
||||
test('empty-but-real input (`< /dev/null`) does not hang', async () => {
|
||||
const run = await runCliWithStdin(['put'], { file: '/dev/null' }, 20_000);
|
||||
expect(run.exited).toBe(true);
|
||||
expect(run.exitCode).toBe(1);
|
||||
}, 30_000);
|
||||
|
||||
test('an empty pipe that closes immediately does not hang', async () => {
|
||||
const run = await runCliWithStdin(['put'], 'closed-empty', 20_000);
|
||||
expect(run.exited).toBe(true);
|
||||
expect(run.exitCode).toBe(1);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('#3513 — applyStdinParam content preservation (subprocess driver)', () => {
|
||||
// Drive the exported helper in a child process so we control the child's
|
||||
// real fd 0 — bun test's own stdin is not a reliable fixture.
|
||||
const DRIVER = `
|
||||
const { applyStdinParam } = await import(${JSON.stringify(CLI)});
|
||||
const op = { name: 'put', params: { content: { type: 'string', required: true } }, cliHints: { stdin: 'content' } };
|
||||
const params = {};
|
||||
await applyStdinParam(op, params);
|
||||
console.log(JSON.stringify(params));
|
||||
process.exit(0);
|
||||
`;
|
||||
|
||||
async function runDriver(
|
||||
stdin: 'hold-open' | 'closed-empty' | { data: string } | { file: string },
|
||||
): Promise<{ exited: boolean; params: Record<string, unknown> | null }> {
|
||||
const proc = Bun.spawn(['bun', '-e', DRIVER], {
|
||||
cwd: REPO,
|
||||
env: { ...process.env, GBRAIN_STDIN_TIMEOUT_MS: '500' },
|
||||
stdin: typeof stdin === 'object' && 'file' in stdin ? Bun.file(stdin.file) : 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
if (typeof stdin === 'object' && 'data' in stdin) {
|
||||
pipeSink(proc).write(stdin.data);
|
||||
await pipeSink(proc).end();
|
||||
} else if (stdin === 'closed-empty') {
|
||||
await pipeSink(proc).end();
|
||||
}
|
||||
let exited = true;
|
||||
const killer = setTimeout(() => {
|
||||
exited = false;
|
||||
try { proc.kill('SIGKILL'); } catch { /* already dead */ }
|
||||
}, 20_000);
|
||||
const [stdout] = await Promise.all([new Response(proc.stdout).text(), proc.exited]);
|
||||
clearTimeout(killer);
|
||||
try { pipeSink(proc).end(); } catch { /* hold-open cleanup */ }
|
||||
const line = stdout.trim().split('\n').pop() ?? '';
|
||||
let params: Record<string, unknown> | null = null;
|
||||
try { params = JSON.parse(line); } catch { /* child killed before printing */ }
|
||||
return { exited, params };
|
||||
}
|
||||
|
||||
test('piped data lands in the stdin param verbatim', async () => {
|
||||
const { exited, params } = await runDriver({ data: '---\ntitle: x\n---\nbody' });
|
||||
expect(exited).toBe(true);
|
||||
expect(params?.content).toBe('---\ntitle: x\n---\nbody');
|
||||
}, 30_000);
|
||||
|
||||
test('/dev/null yields empty-string content (readable, empty — pre-fix parity)', async () => {
|
||||
const { exited, params } = await runDriver({ file: '/dev/null' });
|
||||
expect(exited).toBe(true);
|
||||
expect(params?.content).toBe('');
|
||||
}, 30_000);
|
||||
|
||||
test('empty closed pipe yields empty-string content', async () => {
|
||||
const { exited, params } = await runDriver('closed-empty');
|
||||
expect(exited).toBe(true);
|
||||
expect(params?.content).toBe('');
|
||||
}, 30_000);
|
||||
|
||||
test('held-open pipe times out and leaves the param unset', async () => {
|
||||
const { exited, params } = await runDriver('hold-open');
|
||||
expect(exited).toBe(true); // completes inside the window instead of hanging
|
||||
expect(params).toEqual({});
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
return resolveSearchMode({ mode: 'balanced' });
|
||||
}
|
||||
|
||||
test('KNOBS_HASH_VERSION is 14 (cross-modal still appended; 13→14 compiled_truth boost scope #3430)', () => {
|
||||
test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 embedding-provider migration #3390)', () => {
|
||||
// v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
|
||||
// with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
|
||||
// v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
|
||||
@@ -146,8 +146,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
// v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type
|
||||
// finally reaches asymmetric providers — pre-fix rows were keyed on
|
||||
// document-side query vectors. #2825: 11→12 hard-exclude fold (hx=).
|
||||
// #3430: 13→14 compiled_truth boost no longer applies at detail=medium.
|
||||
expect(KNOBS_HASH_VERSION).toBe(14);
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('flipping unified_multimodal changes the hash', () => {
|
||||
|
||||
@@ -38,7 +38,7 @@ import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { withEnv, emptyHome } from './helpers/with-env.ts';
|
||||
import { runCycle, ALL_PHASES } from '../src/core/cycle.ts';
|
||||
import { mkdtempSync, writeFileSync, rmSync } from 'fs';
|
||||
import { mkdtempSync, writeFileSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
@@ -139,25 +139,19 @@ describe('#2540 (i) — pack omitting optional phases, all enabled phases comple
|
||||
|
||||
describe('#2540 (ii) — an enabled phase that never completes still prevents the stamp', () => {
|
||||
test('every selected phase failing reports status=failed and does NOT stamp last_full_cycle_at', async () => {
|
||||
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
||||
await withEnv({ GBRAIN_HOME: gbrainHome, OPENAI_API_KEY: undefined, ANTHROPIC_API_KEY: undefined }, async () => {
|
||||
await seedSource('always-fails');
|
||||
expect(await readLastFullCycleAt('always-fails')).toBeNull();
|
||||
|
||||
// Deterministic, environment-independent failure: run the sync phase
|
||||
// against a brain directory that no longer exists. The previous shape
|
||||
// ('embed' with OPENAI_API_KEY/ANTHROPIC_API_KEY unset) was
|
||||
// environment-sensitive — on a machine where any OTHER embedding
|
||||
// provider resolves (Voyage, ZeroEntropy, a local endpoint, …), embed
|
||||
// with zero stale chunks succeeds and the cycle reports 'clean',
|
||||
// flipping this test's expectation. A vanished checkout fails the
|
||||
// sync phase on every machine. This is NOT the fix under test; it's
|
||||
// the pre-existing "an enabled phase genuinely never completes" case
|
||||
// the issue says must keep failing doctor's check.
|
||||
rmSync(brainDir, { recursive: true, force: true });
|
||||
// embed is a real, always-enabled phase (no pack gate, no config
|
||||
// .enabled toggle). With no embedding provider key configured it
|
||||
// deterministically fails — this is NOT the fix under test, it's
|
||||
// the pre-existing "an enabled phase genuinely never completes"
|
||||
// case the issue says must keep failing doctor's check.
|
||||
const report = await runCycle(engine, {
|
||||
brainDir,
|
||||
sourceId: 'always-fails',
|
||||
phases: ['sync'],
|
||||
phases: ['embed'],
|
||||
});
|
||||
|
||||
expect(report.status).toBe('failed');
|
||||
|
||||
@@ -79,38 +79,12 @@ describe('doctor checkCycleFreshness', () => {
|
||||
expect(result.message).toMatch(/gbrain dream --source/);
|
||||
});
|
||||
|
||||
test('source with NO last_full_cycle_at (never cycled) returns warn, not fail (#2540)', async () => {
|
||||
// #2540: never-cycled used to FAIL, which turned doctor permanently red
|
||||
// on any install that doesn't cycle every local_path source (e.g. one
|
||||
// nightly `dream --dir <vault>` plus other federated sources) — and on
|
||||
// any source added minutes ago. It surfaces as a warning; only a source
|
||||
// that HAS cycled and then went stale escalates to fail.
|
||||
test('source with NO last_full_cycle_at (never cycled) returns fail', async () => {
|
||||
await engine.executeRaw(`UPDATE sources SET local_path = NULL WHERE id = 'default'`);
|
||||
await seed('virgin');
|
||||
const result = await checkCycleFreshness(engine, { nowMs: NOW });
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toMatch(/never completed a full cycle/);
|
||||
expect(result.message).toMatch(/gbrain dream --source/);
|
||||
});
|
||||
|
||||
test('reporter case (#2540): one cycled vault + never-cycled siblings is warn, not permanent fail', async () => {
|
||||
await engine.executeRaw(`UPDATE sources SET local_path = NULL WHERE id = 'default'`);
|
||||
await seed('nightly-vault', agoH(2)); // the one vault dreamt via --dir
|
||||
await seed('federated-a'); // never cycled
|
||||
await seed('federated-b'); // never cycled
|
||||
const result = await checkCycleFreshness(engine, { nowMs: NOW });
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toMatch(/federated-a/);
|
||||
expect(result.message).toMatch(/federated-b/);
|
||||
expect(result.message).not.toMatch(/nightly-vault/);
|
||||
});
|
||||
|
||||
test('a previously-cycled source gone stale still fails even next to never-cycled sources', async () => {
|
||||
await engine.executeRaw(`UPDATE sources SET local_path = NULL WHERE id = 'default'`);
|
||||
await seed('stale', agoH(72)); // real regression signal
|
||||
await seed('virgin'); // never cycled — warn-only
|
||||
const result = await checkCycleFreshness(engine, { nowMs: NOW });
|
||||
expect(result.status).toBe('fail');
|
||||
expect(result.message).toMatch(/never completed a full cycle/);
|
||||
});
|
||||
|
||||
test('mixed sources: highest severity wins (fail > warn > ok)', async () => {
|
||||
|
||||
@@ -143,10 +143,9 @@ describe('checkEmbeddingWidthConsistency', () => {
|
||||
});
|
||||
|
||||
test('gateway unconfigured: skips with ok', async () => {
|
||||
// Hard-unconfigure so requireConfig() throws — resetGateway() would
|
||||
// restore the preload's test baseline (#3554).
|
||||
const { __unconfigureGatewayForTests } = await import('../src/core/ai/gateway.ts');
|
||||
__unconfigureGatewayForTests();
|
||||
// Reset gateway so requireConfig() throws.
|
||||
const { resetGateway } = await import('../src/core/ai/gateway.ts');
|
||||
resetGateway();
|
||||
const check = await checkEmbeddingWidthConsistency(engine);
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).toContain('gateway not configured');
|
||||
|
||||
@@ -96,28 +96,6 @@ describe('gbrain dream --dir <path> freshness stamp (#1869)', () => {
|
||||
expect(await readLastFullCycleAt('mothballed')).toBeNull();
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
test('an ARCHIVED alias of the same path does not shadow the active source (#2540)', async () => {
|
||||
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
||||
// Ordinary shape: a source was archived and re-added under a new id
|
||||
// pointing at the same checkout. Seed the archived twin FIRST so a
|
||||
// filterless `LIMIT 1` scan finds it first.
|
||||
await seedSource('retired-twin', true);
|
||||
await seedSource('active-twin', false);
|
||||
|
||||
const report = await runDream(engine, ['--dir', brainDir, '--phase', 'lint', '--json']);
|
||||
expect(report).toBeTruthy();
|
||||
if (report) expect(['ok', 'clean']).toContain(report.status);
|
||||
|
||||
// Pre-fix, resolveSourceForDir's exact match had no `archived = false`
|
||||
// filter and no ORDER BY, so the archived twin won the lookup; dream's
|
||||
// archived guard then (correctly) refused to stamp it — and the ACTIVE
|
||||
// source silently never got its stamp, leaving doctor's cycle_freshness
|
||||
// permanently stale on a healthy install.
|
||||
expect(await readLastFullCycleAt('active-twin')).not.toBeNull();
|
||||
expect(await readLastFullCycleAt('retired-twin')).toBeNull();
|
||||
});
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
/**
|
||||
|
||||
@@ -252,87 +252,6 @@ describe('upsertChunks — model provenance uses gateway-resolved model, not com
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured — it
|
||||
// never returns falsy — so the reland's `|| resolvedModel` guard was dead
|
||||
// code and the catch path still stamped the compile-time default onto rows
|
||||
// whose vectors came from the config-resolved provider. The engine must
|
||||
// fall back to the brain's own `config.embedding_model` row instead.
|
||||
test('#3461: unconfigured gateway falls back to the brain config model, never the compiled default', async () => {
|
||||
await engine.setConfig('embedding_model', 'voyage:voyage-3-large');
|
||||
// The preload's beforeEach re-configures the gateway before every test,
|
||||
// so the reset must happen INSIDE the test body.
|
||||
resetGateway();
|
||||
|
||||
await engine.putPage('docs/provenance-throw-path', {
|
||||
type: 'concept',
|
||||
title: 'Provenance throw-path page',
|
||||
compiled_truth: 'Chunk written while the gateway is unconfigured.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-throw-path', [
|
||||
{ chunk_index: 0, chunk_text: 'throw-path provenance chunk', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string }>(
|
||||
`SELECT cc.model FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-throw-path'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].model).toBe('voyage:voyage-3-large');
|
||||
|
||||
// Restore the value initSchema wrote for the rest of the file.
|
||||
await engine.setConfig('embedding_model', 'openai:text-embedding-3-large');
|
||||
});
|
||||
|
||||
// #3461 sibling: on a partial re-upsert that carries NO new embedding (the
|
||||
// exact shape `embed --stale` produces for a page's non-stale chunks), the
|
||||
// preserved vector must KEEP its original model label. The old
|
||||
// COALESCE(EXCLUDED.model, …) relabeled it with the current gateway model.
|
||||
test('#3461: preserved vector keeps its original model label on a no-embedding re-upsert', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { OPENAI_API_KEY: 'sk-test' },
|
||||
});
|
||||
|
||||
await engine.putPage('docs/provenance-preserve', {
|
||||
type: 'concept',
|
||||
title: 'Provenance preserve page',
|
||||
compiled_truth: 'Chunk embedded under model A, re-upserted under model B.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'stable chunk text',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(VEC1536_A),
|
||||
},
|
||||
]);
|
||||
|
||||
// Model swap: the gateway now resolves a different model, and the
|
||||
// re-upsert (same chunk_text) carries no new embedding.
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { VOYAGE_API_KEY: 'test' },
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{ chunk_index: 0, chunk_text: 'stable chunk text', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string; has_embedding: boolean }>(
|
||||
`SELECT cc.model, cc.embedding IS NOT NULL AS has_embedding
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-preserve'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].has_embedding).toBe(true); // vector preserved…
|
||||
expect(rows[0].model).toBe('openai:text-embedding-3-large'); // …and its label still describes it
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildVectorCastFragment — engine SQL composer (D3)', () => {
|
||||
|
||||
@@ -26,12 +26,8 @@ if (skip) {
|
||||
}
|
||||
|
||||
describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
// 60s hook budget: setupDB runs connect + the full migration chain, which
|
||||
// exceeds bun's default 5s hook timeout on loaded CI runners. Hooks do NOT
|
||||
// inherit a test's third-arg timeout (verified on bun 1.3.14) — they need
|
||||
// their own second-arg budget. Same pattern as op-checkpoint-jsonb-parity.
|
||||
beforeAll(async () => { await setupDB(); }, 60_000);
|
||||
afterAll(async () => { await teardownDB(); }, 60_000);
|
||||
beforeAll(async () => { await setupDB(); });
|
||||
afterAll(async () => { await teardownDB(); });
|
||||
|
||||
test('putPage writes frontmatter as object, not double-encoded string', async () => {
|
||||
const engine = getEngine();
|
||||
@@ -73,7 +69,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
`;
|
||||
expect(row.t).toBe('object');
|
||||
expect(row.marker).toBe('rawdata-value');
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
test('logIngest writes pages_updated as array, not double-encoded string', async () => {
|
||||
const engine = getEngine();
|
||||
@@ -95,7 +91,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
expect(row.t).toBe('array');
|
||||
expect(Number(row.n)).toBe(3);
|
||||
expect(row.first).toBe('test/a');
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
// files.ts:254 (uploadRaw's cloud-upload branch) was changed from
|
||||
// `${JSON.stringify({...})}::jsonb` to `${sql.json({...})}` in v0.12.1.
|
||||
@@ -118,7 +114,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
expect(row.t).toBe('object');
|
||||
expect(row.type).toBe('pdf');
|
||||
expect(row.method).toBe('TUS resumable');
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
// Source-level tripwire: if anyone re-introduces the old `${JSON.stringify(x)}::jsonb`
|
||||
// pattern for the fixed sites, fail loudly. Greps actual source files per the
|
||||
@@ -133,5 +129,5 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
const source = await Bun.file(new URL(rel, import.meta.url)).text();
|
||||
expect(source.match(bad)?.[0] ?? null).toBeNull();
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -81,11 +81,6 @@ function copyFixturesIntoTempWorkspace(): Workspace {
|
||||
|
||||
let workspace: Workspace;
|
||||
|
||||
// Restore (not delete) after each test: the audit-dir preload sets
|
||||
// GBRAIN_AUDIT_DIR once at process start, and deleting it leaks the
|
||||
// operator's real ~/.gbrain/audit/ to every later file in the shard.
|
||||
const priorAuditDir = process.env.GBRAIN_AUDIT_DIR;
|
||||
|
||||
beforeEach(() => {
|
||||
workspace = copyFixturesIntoTempWorkspace();
|
||||
// Redirect audit dir to the tempdir so the snapshot file doesn't pollute
|
||||
@@ -94,8 +89,7 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (priorAuditDir === undefined) delete process.env.GBRAIN_AUDIT_DIR;
|
||||
else process.env.GBRAIN_AUDIT_DIR = priorAuditDir;
|
||||
delete process.env.GBRAIN_AUDIT_DIR;
|
||||
workspace.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -6,11 +6,7 @@
|
||||
* process.env.
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__unconfigureGatewayForTests,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
|
||||
import {
|
||||
validateEmbeddingCreds,
|
||||
formatEmbeddingCredsError,
|
||||
@@ -26,12 +22,18 @@ import type { AIGatewayConfig } from '../src/core/ai/types.ts';
|
||||
// isAvailable('embedding') check. That's what made facts-backstop-gating
|
||||
// fail intermittently (bin-pack-dependent) on CI shard 10.
|
||||
//
|
||||
// #3554: resetGateway() now restores the preload's legacy pin itself (the
|
||||
// preload registers it via __setGatewayResetBaselineForTests), so a bare
|
||||
// reset is safe here — the NEXT file's beforeAll sees the 1536-d baseline,
|
||||
// not a null gateway that would seed 1280-d schemas under 1536-d fixtures.
|
||||
// Don't end on a bare resetGateway() either: the NEXT file's beforeAll
|
||||
// (often engine.initSchema, which sizes vector columns from ambient gateway
|
||||
// state) runs before the legacy-embedding-preload's per-test restore, so a
|
||||
// null gateway here would seed 1280-d schemas under 1536-d fixtures.
|
||||
// Restore the preload's legacy pin instead.
|
||||
afterAll(() => {
|
||||
resetGateway();
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ...process.env },
|
||||
});
|
||||
});
|
||||
|
||||
function baseConfig(overrides: Partial<AIGatewayConfig> = {}): AIGatewayConfig {
|
||||
@@ -136,9 +138,7 @@ describe('validateEmbeddingCreds', () => {
|
||||
});
|
||||
|
||||
test('throws no_gateway_config when gateway was not configured', () => {
|
||||
// resetGateway() restores the preload's test baseline (#3554), so this
|
||||
// test needs the hard variant to get a genuinely unconfigured gateway.
|
||||
__unconfigureGatewayForTests();
|
||||
// resetGateway() in beforeEach already cleared _config.
|
||||
let caught: unknown;
|
||||
try { validateEmbeddingCreds(); } catch (e) { caught = e; }
|
||||
expect(caught).toBeInstanceOf(EmbeddingCredentialError);
|
||||
|
||||
@@ -277,79 +277,3 @@ describe('embedStaleForSource', () => {
|
||||
expect(txtRow.embedded_at).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — re-embed must reproduce the page's STORED contextual-retrieval
|
||||
// wrapping convention. Before the fix, every plain re-embed (including the
|
||||
// normal post-model-migration `embed --stale`) embedded raw chunk_text,
|
||||
// silently replacing context-wrapped vectors with unwrapped ones.
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('contextual-retrieval wrapping on re-embed (#3507)', () => {
|
||||
/** embedFn that records every text it is asked to embed. */
|
||||
function capturingEmbedFn(seen: string[]) {
|
||||
return (texts: string[]): Promise<Float32Array[]> => {
|
||||
seen.push(...texts);
|
||||
return fakeEmbedFn(texts);
|
||||
};
|
||||
}
|
||||
|
||||
async function seedWrappablePage(slug: string, title: string): Promise<void> {
|
||||
await engine.putPage(slug, { type: 'note', title, compiled_truth: 'seeded' });
|
||||
await engine.upsertChunks(slug, [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk about widgets', chunk_source: 'compiled_truth', token_count: 4 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', token_count: 4 },
|
||||
]);
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed sends title-wrapped texts; fenced_code stays raw', async () => {
|
||||
await seedWrappablePage('wrapped-page', 'Widget Notes');
|
||||
await engine.updatePageContextualRetrievalState('wrapped-page', 'default', 'title', 'gen-title');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk about widgets');
|
||||
expect(seen).toContain('const x = 1;'); // fenced_code is NEVER wrapped (D20-T4)
|
||||
|
||||
// D20-T1: the canonical chunk_text is NOT rewritten — wrapping is embed-input-only.
|
||||
const chunks = await engine.getChunks('wrapped-page');
|
||||
expect(chunks.map((c) => c.chunk_text).sort()).toEqual(['const x = 1;', 'prose chunk about widgets']);
|
||||
// Mode stamp unchanged for title-tier pages.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'wrapped-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: re-embed applies the title-tier wrapper and restamps honestly', async () => {
|
||||
await seedWrappablePage('synopsis-page', 'Synopsis Notes');
|
||||
await engine.updatePageContextualRetrievalState('synopsis-page', 'default', 'per_chunk_synopsis', 'gen-synopsis');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
// Synopsis re-generation is a paid backfill concern; the plain re-embed
|
||||
// lands at the title tier (the service's own D14 fallback tier)…
|
||||
expect(seen).toContain('<context>Synopsis Notes\n</context>\nprose chunk about widgets');
|
||||
// …and the stamped mode is updated so it keeps describing the vectors.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'synopsis-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('unstamped page (NULL mode) embeds raw chunk_text — convention preserved', async () => {
|
||||
await seedWrappablePage('plain-page', 'Plain Notes');
|
||||
// No updatePageContextualRetrievalState call: pre-CR page.
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('prose chunk about widgets');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -907,76 +907,3 @@ describe('runEmbed preserves code-chunk metadata across re-embed (regression for
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — `embed --stale` must reproduce the page's STORED
|
||||
// contextual-retrieval wrapping convention instead of embedding raw
|
||||
// chunk_text (which silently stripped contextual prefixes on every
|
||||
// re-embed, including the normal post-model-migration path).
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('embed --stale contextual-retrieval wrapping (#3507)', () => {
|
||||
const wrapChunks = [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth', embedded_at: null, token_count: 1 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', embedded_at: null, token_count: 1 },
|
||||
];
|
||||
const wrapStale = [
|
||||
{ slug: 'wrapped', chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth' as const, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
{ slug: 'wrapped', chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code' as any, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
];
|
||||
|
||||
function wrappingHarness(mode: string | null) {
|
||||
const seen: string[] = [];
|
||||
const restamps: any[][] = [];
|
||||
embedBatchBehavior = async (texts: string[]) => {
|
||||
seen.push(...texts);
|
||||
return texts.map(() => new Float32Array(1536));
|
||||
};
|
||||
const engine = mockEngine({
|
||||
countStaleChunks: async () => 2,
|
||||
listStaleChunks: async () => wrapStale,
|
||||
getPage: async () => ({
|
||||
slug: 'wrapped',
|
||||
title: 'Widget Notes',
|
||||
source_id: 'default',
|
||||
compiled_truth: 'x',
|
||||
timeline: '',
|
||||
contextual_retrieval_mode: mode,
|
||||
}),
|
||||
getChunks: async () => wrapChunks,
|
||||
upsertChunks: async () => {},
|
||||
updatePageContextualRetrievalState: async (...args: any[]) => { restamps.push(args); },
|
||||
});
|
||||
return { engine, seen, restamps };
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed wraps prose with the title prefix; fenced_code stays raw', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('title');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(seen).toContain('const x = 1;');
|
||||
expect(restamps).toHaveLength(0); // title tier: stamp already honest
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: fully re-embedded page restamps to the title tier', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('per_chunk_synopsis');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(restamps).toHaveLength(1);
|
||||
const [slug, sourceId, newMode] = restamps[0];
|
||||
expect(slug).toBe('wrapped');
|
||||
expect(sourceId).toBe('default');
|
||||
expect(newMode).toBe('title');
|
||||
});
|
||||
|
||||
test('page with no stored CR mode embeds raw chunk_text (convention preserved)', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness(null);
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('prose chunk');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
expect(restamps).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
/**
|
||||
* Pins the Memvelope envelope importer contract: deterministic markdown output,
|
||||
* provenance frontmatter, citation-bearing bodies, and loud collision handling.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
// The same parser gbrain uses to ingest frontmatter (src/core/markdown.ts), so
|
||||
// the injection test asserts against the real consumer rather than a substring.
|
||||
import { safeLoad as yamlSafeLoad } from 'js-yaml';
|
||||
|
||||
const SCRIPT_PATH = join(import.meta.dir, '..', 'scripts', 'envelope-to-gbrain.mjs');
|
||||
const FIXTURE_PATH = join(import.meta.dir, 'fixtures', 'memvelope', 'sample.mve.json');
|
||||
const TEMP_DIRS: string[] = [];
|
||||
|
||||
afterAll(() => {
|
||||
for (const dir of TEMP_DIRS) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function tempDir(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'envelope-to-gbrain-'));
|
||||
TEMP_DIRS.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
async function runImporter(envelopePath: string, outDir = tempDir()) {
|
||||
// The script is plain Node-compatible ESM; Bun can execute it directly in CI
|
||||
// without requiring a separate node toolchain.
|
||||
const proc = Bun.spawn([process.execPath, SCRIPT_PATH, envelopePath, outDir], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
await proc.exited;
|
||||
const stdout = await new Response(proc.stdout).text();
|
||||
const stderr = await new Response(proc.stderr).text();
|
||||
return { exitCode: proc.exitCode, stdout, stderr, outDir };
|
||||
}
|
||||
|
||||
function markdownFiles(dir: string): string[] {
|
||||
return readdirSync(dir).filter((name) => name.endsWith('.md')).sort();
|
||||
}
|
||||
|
||||
function readOnlyMarkdown(dir: string): string {
|
||||
const files = markdownFiles(dir);
|
||||
expect(files).toHaveLength(1);
|
||||
return readFileSync(join(dir, files[0]), 'utf8');
|
||||
}
|
||||
|
||||
describe('envelope-to-gbrain importer', () => {
|
||||
test('sample envelope writes exactly one markdown page and reports count', async () => {
|
||||
const result = await runImporter(FIXTURE_PATH);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(markdownFiles(result.outDir)).toHaveLength(1);
|
||||
expect(result.stdout).toContain('wrote 1 markdown page(s)');
|
||||
});
|
||||
|
||||
test('filename is keyed by conversation id with date prefix', async () => {
|
||||
const result = await runImporter(FIXTURE_PATH);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-c-3f9a2b.md']);
|
||||
});
|
||||
|
||||
test('frontmatter carries conversation provenance fields', async () => {
|
||||
const result = await runImporter(FIXTURE_PATH);
|
||||
const page = readOnlyMarkdown(result.outDir);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(page).toContain('type: conversation');
|
||||
expect(page).toContain('title: "Onboarding Checklist Draft"');
|
||||
expect(page).toContain('date: 2025-11-02');
|
||||
expect(page).toContain('source: "chatgpt"');
|
||||
expect(page).toContain('memvelope_conversation_id: "c-3f9a2b"');
|
||||
expect(page).toContain('origin: memvelope/envelope-v0');
|
||||
});
|
||||
|
||||
test('body carries role labels and message-id citations', async () => {
|
||||
const result = await runImporter(FIXTURE_PATH);
|
||||
const page = readOnlyMarkdown(result.outDir);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(page).toContain('· m1');
|
||||
expect(page).toContain('· m4');
|
||||
expect(page).toContain('**Me**');
|
||||
expect(page).toContain('**Assistant**');
|
||||
});
|
||||
|
||||
test('output is deterministic across repeated runs', async () => {
|
||||
const first = await runImporter(FIXTURE_PATH);
|
||||
const second = await runImporter(FIXTURE_PATH);
|
||||
|
||||
expect(first.exitCode).toBe(0);
|
||||
expect(second.exitCode).toBe(0);
|
||||
expect(readOnlyMarkdown(first.outDir)).toBe(readOnlyMarkdown(second.outDir));
|
||||
});
|
||||
|
||||
test('duplicate conversation ids warn and report distinct files written', async () => {
|
||||
const inputDir = tempDir();
|
||||
const envelopePath = join(inputDir, 'duplicate.mve.json');
|
||||
writeFileSync(envelopePath, JSON.stringify({
|
||||
memvelope: 'envelope-v0',
|
||||
meta: { source_provider: 'chatgpt' },
|
||||
conversations: [
|
||||
{
|
||||
id: 'c-repeat',
|
||||
title: 'First repeated id',
|
||||
created_at: '2025-11-02T14:22:51.000Z',
|
||||
messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example noted the first checklist draft.' }],
|
||||
},
|
||||
{
|
||||
id: 'c-repeat',
|
||||
title: 'Second repeated id',
|
||||
created_at: '2025-11-02T15:22:51.000Z',
|
||||
messages: [{ id: 'm2', role: 'assistant', ts: '2025-11-02T15:22:51.000Z', text: 'Assistant noted the repeated id collision.' }],
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
const result = await runImporter(envelopePath);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stderr).toContain('warning: filename collision on "2025-11-02-c-repeat.md"');
|
||||
expect(result.stdout).toContain('wrote 1 markdown page(s)');
|
||||
expect(markdownFiles(result.outDir)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('missing or foreign format is rejected', async () => {
|
||||
const inputDir = tempDir();
|
||||
const envelopePath = join(inputDir, 'not-envelope.json');
|
||||
writeFileSync(envelopePath, JSON.stringify({ conversations: [] }));
|
||||
|
||||
const result = await runImporter(envelopePath);
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.stderr).toContain('envelope-v0');
|
||||
});
|
||||
|
||||
test('missing conversation id uses positional fallback filename', async () => {
|
||||
const inputDir = tempDir();
|
||||
const envelopePath = join(inputDir, 'missing-id.mve.json');
|
||||
writeFileSync(envelopePath, JSON.stringify({
|
||||
memvelope: 'envelope-v0',
|
||||
meta: { source_provider: 'chatgpt' },
|
||||
conversations: [
|
||||
{
|
||||
title: 'Missing id example',
|
||||
created_at: '2025-11-02T14:22:51.000Z',
|
||||
messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example asked for a fallback filename.' }],
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
const result = await runImporter(envelopePath);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-conv-1.md']);
|
||||
});
|
||||
|
||||
test('missing conversation id omits the provenance key rather than emitting a value', async () => {
|
||||
const inputDir = tempDir();
|
||||
const envelopePath = join(inputDir, 'missing-id-frontmatter.mve.json');
|
||||
writeFileSync(envelopePath, JSON.stringify({
|
||||
memvelope: 'envelope-v0',
|
||||
meta: { source_provider: 'chatgpt' },
|
||||
conversations: [
|
||||
{
|
||||
title: 'Missing id example',
|
||||
created_at: '2025-11-02T14:22:51.000Z',
|
||||
messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example asked about frontmatter.' }],
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
const result = await runImporter(envelopePath);
|
||||
const page = readOnlyMarkdown(result.outDir);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
// Absent means absent: never the literal string `undefined`, and never the
|
||||
// positional filename fallback masquerading as a real conversation id.
|
||||
expect(page).not.toContain('memvelope_conversation_id');
|
||||
expect(page).not.toContain('undefined');
|
||||
expect(page).toContain('source: "chatgpt"');
|
||||
});
|
||||
|
||||
test('a provider string carrying a newline cannot inject frontmatter keys', async () => {
|
||||
const inputDir = tempDir();
|
||||
const envelopePath = join(inputDir, 'injecting-provider.mve.json');
|
||||
writeFileSync(envelopePath, JSON.stringify({
|
||||
memvelope: 'envelope-v0',
|
||||
meta: { source_provider: 'chatgpt\ntype: injected\nowner: attacker' },
|
||||
conversations: [
|
||||
{
|
||||
id: 'c-inject',
|
||||
title: 'Injection attempt',
|
||||
created_at: '2025-11-02T14:22:51.000Z',
|
||||
messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example sent a hostile provider string.' }],
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
const result = await runImporter(envelopePath);
|
||||
const page = readOnlyMarkdown(result.outDir);
|
||||
const frontmatter = page.split('---')[1] ?? '';
|
||||
const parsed = yamlSafeLoad(frontmatter) as Record<string, unknown>;
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
// The newline is escaped inside a quoted scalar, so the hostile text stays
|
||||
// one value instead of becoming keys. Asserted structurally: a substring
|
||||
// check cannot tell a real key from the same characters inside a quoted
|
||||
// value, and would pass for the wrong reason.
|
||||
expect(Object.keys(parsed).sort()).toEqual([
|
||||
'date',
|
||||
'memvelope_conversation_id',
|
||||
'origin',
|
||||
'source',
|
||||
'title',
|
||||
'type',
|
||||
]);
|
||||
expect(parsed.type).toBe('conversation');
|
||||
expect(parsed.source).toBe('chatgpt\ntype: injected\nowner: attacker');
|
||||
});
|
||||
});
|
||||
@@ -672,51 +672,6 @@ describe('runExtractFacts — empty-fence guard (Codex R2-#7)', () => {
|
||||
});
|
||||
|
||||
describe('runExtractFacts — multi-source isolation', () => {
|
||||
test('a pending legacy row in source A does NOT jam extraction for source B (#2646 source-scope)', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO sources (id, name, config) VALUES ('work', 'work', '{}'::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
);
|
||||
|
||||
// Source "work": a genuine pending legacy row (row_num NULL, active,
|
||||
// live backing page) — the exact shape that must gate work's cycle.
|
||||
await engine.putPage('people/alice', {
|
||||
title: 'people/alice', type: 'person',
|
||||
compiled_truth: FACT_FENCE(`| 1 | work fence fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`),
|
||||
frontmatter: {}, timeline: '',
|
||||
}, { sourceId: 'work' });
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability,
|
||||
valid_from, source, confidence)
|
||||
VALUES ('work', 'people/alice', 'work legacy claim', 'fact', 'private', 'medium',
|
||||
now(), 'mcp:put_page', 1.0)`,
|
||||
);
|
||||
|
||||
// Source "default": clean — no legacy rows, one fenced page.
|
||||
await putPage('people/bob', FACT_FENCE(
|
||||
`| 1 | default fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
|
||||
// default's run must NOT be jammed by work's pending backlog.
|
||||
const rDefault = await runExtractFacts(engine, { slugs: ['people/bob'], sourceId: 'default' });
|
||||
expect(rDefault.guardTriggered).toBe(false);
|
||||
expect(rDefault.legacyRowsPending).toBe(0);
|
||||
expect(rDefault.factsInserted).toBe(1);
|
||||
|
||||
// work's own run still gates (discriminator stays sharp).
|
||||
const rWork = await runExtractFacts(engine, { slugs: ['people/alice'], sourceId: 'work' });
|
||||
expect(rWork.guardTriggered).toBe(true);
|
||||
expect(rWork.legacyRowsPending).toBe(1);
|
||||
expect(rWork.factsInserted).toBe(0);
|
||||
// The drain advice must be one that actually re-runs Phase B — a bare
|
||||
// `apply-migrations --yes` no-ops once the ledger says complete.
|
||||
expect(rWork.warnings.some(w => w.includes('--force-retry 0.32.2'))).toBe(true);
|
||||
expect(rWork.warnings.some(w => w.includes('forget_fact'))).toBe(true);
|
||||
expect(rWork.warnings.some(w => w.includes('source "work"'))).toBe(true);
|
||||
});
|
||||
|
||||
test('deleteFactsForPage scoping does not affect other sources', async () => {
|
||||
// Seed sources work + home.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
writeReceipt,
|
||||
type ExtractReceiptInput,
|
||||
} from '../../src/core/extract/receipt-writer.ts';
|
||||
import { slugifySegment } from '../../src/core/sync.ts';
|
||||
|
||||
const BASE_INPUT: ExtractReceiptInput = {
|
||||
kind: 'facts.conversation',
|
||||
@@ -82,31 +81,6 @@ describe('shortRunId / dateFromIso — pure helpers', () => {
|
||||
expect(shortRunId('op_check_abc')).toBe('op_check');
|
||||
});
|
||||
|
||||
// #3443 — a short form ending in '-' (e.g. propose-<timestamp> run ids)
|
||||
// desynced the DB receipt slug from its Git-backed slug: slugifySegment()
|
||||
// strips boundary hyphens during repo sync, so the write-through created a
|
||||
// normalized sibling instead of materializing the existing page.
|
||||
test('shortRunId is canonical under slugifySegment for every receipt-producing run-id family (#3443)', () => {
|
||||
const familyRunIds = [
|
||||
'propose-20260724103000-ab12cd34', // cycle/propose-takes.ts
|
||||
`atoms-${Date.now().toString(36)}-pers`, // cycle/extract-atoms.ts
|
||||
`efacts-${Date.now().toString(36)}-pers`, // cycle/extract-facts.ts
|
||||
`concepts-${Date.now().toString(36)}`, // cycle/synthesize-concepts.ts
|
||||
`ecf-${Date.now().toString(36)}-pers`, // extract-conversation-facts.ts
|
||||
];
|
||||
for (const runId of familyRunIds) {
|
||||
const short = shortRunId(runId);
|
||||
expect(slugifySegment(short)).toBe(short);
|
||||
expect(short.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('shortRunId trims boundary hyphens introduced by truncation', () => {
|
||||
expect(shortRunId('propose-20260724103000-ab12cd34')).toBe('propose');
|
||||
// Pathological all-separator prefix still yields a non-empty segment.
|
||||
expect(shortRunId('--------tail')).toBe('run');
|
||||
});
|
||||
|
||||
test('dateFromIso extracts YYYY-MM-DD prefix', () => {
|
||||
expect(dateFromIso('2026-05-27T14:30:00Z')).toBe('2026-05-27');
|
||||
expect(dateFromIso('2026-05-27T14:30:00.123456Z')).toBe('2026-05-27');
|
||||
|
||||
+1
-20
@@ -4,7 +4,7 @@ import { join, basename } from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import { extname } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { collectFiles, formatFileSizeKb } from '../src/commands/files.ts';
|
||||
import { collectFiles } from '../src/commands/files.ts';
|
||||
import { operationsByName } from '../src/core/operations.ts';
|
||||
import * as db from '../src/core/db.ts';
|
||||
|
||||
@@ -51,25 +51,6 @@ afterAll(() => {
|
||||
rmSync(TMP, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('formatFileSizeKb', () => {
|
||||
test('formats number, bigint, and string database values', () => {
|
||||
expect(formatFileSizeKb(35 * 1024)).toBe('35KB');
|
||||
expect(formatFileSizeKb(35n * 1024n)).toBe('35KB');
|
||||
expect(formatFileSizeKb('35840')).toBe('35KB');
|
||||
});
|
||||
|
||||
test('preserves zero-byte files instead of reporting an unknown size', () => {
|
||||
expect(formatFileSizeKb(0)).toBe('0KB');
|
||||
expect(formatFileSizeKb(0n)).toBe('0KB');
|
||||
});
|
||||
|
||||
test('reports missing or invalid sizes as unknown', () => {
|
||||
expect(formatFileSizeKb(null)).toBe('?');
|
||||
expect(formatFileSizeKb('not-a-number')).toBe('?');
|
||||
expect(formatFileSizeKb(-1)).toBe('?');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMimeType', () => {
|
||||
test('returns correct MIME for .jpg', () => {
|
||||
expect(getMimeType('photo.jpg')).toBe('image/jpeg');
|
||||
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"memvelope": "envelope-v0",
|
||||
"meta": {
|
||||
"source_provider": "chatgpt",
|
||||
"conversation_count": 1,
|
||||
"message_count": 4
|
||||
},
|
||||
"conversations": [
|
||||
{
|
||||
"id": "c-3f9a2b",
|
||||
"title": "Onboarding Checklist Draft",
|
||||
"created_at": "2025-11-02T14:22:51.000Z",
|
||||
"updated_at": "2025-11-02T14:31:12.000Z",
|
||||
"messages": [
|
||||
{
|
||||
"id": "m1",
|
||||
"role": "user",
|
||||
"ts": "2025-11-02T14:22:51.000Z",
|
||||
"text": "alice-example is drafting acme-example's widget-co onboarding checklist and wants a concise first pass."
|
||||
},
|
||||
{
|
||||
"id": "m2",
|
||||
"role": "assistant",
|
||||
"ts": "2025-11-02T14:24:03.000Z",
|
||||
"text": "Start with account setup, workspace access, sample widget review, and a first-week check-in with the acme-example owner."
|
||||
},
|
||||
{
|
||||
"id": "m3",
|
||||
"role": "user",
|
||||
"ts": "2025-11-02T14:28:19.000Z",
|
||||
"text": "Add a note that bob-example should compare fund-a and fund-b reporting needs before the kickoff."
|
||||
},
|
||||
{
|
||||
"id": "m4",
|
||||
"role": "assistant",
|
||||
"ts": "2025-11-02T14:31:12.000Z",
|
||||
"text": "Include a pre-kickoff step for bob-example to list fund-a and fund-b reporting questions, then confirm owners with charlie-example."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -9,11 +9,7 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
__unconfigureGatewayForTests,
|
||||
isAvailable,
|
||||
resetGateway,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import { isAvailable, resetGateway } from '../src/core/ai/gateway.ts';
|
||||
import { runExtractConversationFacts } from '../src/commands/extract-conversation-facts.ts';
|
||||
import { runEnrich } from '../src/commands/enrich.ts';
|
||||
|
||||
@@ -31,10 +27,7 @@ beforeEach(() => {
|
||||
}));
|
||||
process.env.GBRAIN_HOME = home;
|
||||
process.env.OPENAI_API_KEY = 'test-key';
|
||||
// Hard-unconfigure: this suite exists to exercise the COLD-gateway path
|
||||
// (#2590), and resetGateway() now restores the preload's test baseline
|
||||
// (#3554), which would make configureGatewayIfUninitialized a no-op.
|
||||
__unconfigureGatewayForTests();
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
/**
|
||||
* #1305 — getHealth() must exclude soft-deleted pages from every
|
||||
* page-scoped count, the same posture getStats() has had since v0.26.5.
|
||||
*
|
||||
* Pre-fix, getHealth counted raw `pages` rows: page_count and orphan_pages
|
||||
* included soft-deleted pages, the entity_pages CTE kept deleted entities in
|
||||
* the link/timeline coverage denominators and in most_connected, and
|
||||
* brain_score therefore never moved when a user soft-deleted pages.
|
||||
*
|
||||
* Boundary (deliberate): chunk- and link-scoped counts (embed_coverage,
|
||||
* missing_embeddings, link_count, dead_links) stay RAW — they occupy storage
|
||||
* until the autopilot purge phase, matching getStats. Destructive-removal
|
||||
* counts (purge paths, #2235) also deliberately count all rows and are
|
||||
* untouched here.
|
||||
*
|
||||
* Runs against PGLite — the fixed SQL shapes are identical in both engines.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
for (const t of ['links', 'content_chunks', 'timeline_entries', 'tags', 'page_versions', 'pages']) {
|
||||
await (engine as any).db.exec(`DELETE FROM ${t}`);
|
||||
}
|
||||
});
|
||||
|
||||
async function seedNote(slug: string): Promise<void> {
|
||||
await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `content of ${slug}`, frontmatter: {} });
|
||||
}
|
||||
|
||||
async function pageId(slug: string): Promise<number> {
|
||||
return (await (engine as any).db.query(`SELECT id FROM pages WHERE slug=$1`, [slug])).rows[0].id;
|
||||
}
|
||||
|
||||
describe('#1305 — getHealth excludes soft-deleted pages', () => {
|
||||
test('page_count and orphan_pages match getStats after soft-delete (the issue repro)', async () => {
|
||||
for (let i = 0; i < 10; i++) await seedNote(`wiki/note-${i}`);
|
||||
for (let i = 0; i < 6; i++) await engine.softDeletePage(`wiki/note-${i}`);
|
||||
|
||||
const stats = await engine.getStats();
|
||||
const health = await engine.getHealth();
|
||||
expect(stats.page_count).toBe(4);
|
||||
// Pre-fix: 10 (raw rows). getHealth must agree with getStats.
|
||||
expect(health.page_count).toBe(4);
|
||||
// Pre-fix: 10 — deleted pages stayed in the islanded scan.
|
||||
expect(health.orphan_pages).toBe(4);
|
||||
});
|
||||
|
||||
test('brain_score moves when the user soft-deletes the islanded pages', async () => {
|
||||
// 2 connected pages + 8 islanded ones.
|
||||
await seedNote('wiki/hub');
|
||||
await seedNote('wiki/leaf');
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO links (from_page_id, to_page_id, link_type) VALUES ($1, $2, 'mentions')`,
|
||||
[await pageId('wiki/hub'), await pageId('wiki/leaf')],
|
||||
);
|
||||
for (let i = 0; i < 8; i++) await seedNote(`wiki/clutter-${i}`);
|
||||
|
||||
const before = await engine.getHealth();
|
||||
for (let i = 0; i < 8; i++) await engine.softDeletePage(`wiki/clutter-${i}`);
|
||||
const after = await engine.getHealth();
|
||||
|
||||
// Pre-fix both assertions fail: orphan_pages stayed 8 and brain_score
|
||||
// was byte-identical before/after the delete.
|
||||
expect(after.orphan_pages).toBe(0);
|
||||
expect(after.brain_score).toBeGreaterThan(before.brain_score);
|
||||
});
|
||||
|
||||
test('entity coverage denominators and most_connected exclude deleted entities', async () => {
|
||||
// Live entity: inbound link + timeline entry → full coverage.
|
||||
await engine.putPage('people/alice-example', { type: 'person', title: 'Alice', compiled_truth: 'a person', frontmatter: {} });
|
||||
await engine.putPage('people/bob-example', { type: 'person', title: 'Bob', compiled_truth: 'another person', frontmatter: {} });
|
||||
await seedNote('wiki/mentions-alice');
|
||||
const aliceId = await pageId('people/alice-example');
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO links (from_page_id, to_page_id, link_type) VALUES ($1, $2, 'mentions')`,
|
||||
[await pageId('wiki/mentions-alice'), aliceId],
|
||||
);
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO timeline_entries (page_id, date, summary) VALUES ($1, '2026-01-01', 'met alice')`,
|
||||
[aliceId],
|
||||
);
|
||||
|
||||
await engine.softDeletePage('people/bob-example');
|
||||
const h = await engine.getHealth();
|
||||
|
||||
// Pre-fix: bob stayed in the entity_pages CTE → coverage 0.5 each,
|
||||
// and bob appeared in most_connected.
|
||||
expect(h.link_coverage).toBe(1);
|
||||
expect(h.timeline_coverage).toBe(1);
|
||||
expect(h.most_connected.map((c) => c.slug)).not.toContain('people/bob-example');
|
||||
});
|
||||
|
||||
test('chunk storage counts stay raw (the deliberate boundary)', async () => {
|
||||
await seedNote('wiki/kept');
|
||||
await seedNote('wiki/gone');
|
||||
for (const slug of ['wiki/kept', 'wiki/gone']) {
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO content_chunks (page_id, chunk_index, chunk_text) VALUES ($1, 0, 'chunk')`,
|
||||
[await pageId(slug)],
|
||||
);
|
||||
}
|
||||
await engine.softDeletePage('wiki/gone');
|
||||
|
||||
const h = await engine.getHealth();
|
||||
// Soft-deleted pages' chunks still occupy storage until purge; the
|
||||
// missing_embeddings count keeps seeing them, same as getStats.
|
||||
expect(h.missing_embeddings).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -18,11 +18,7 @@
|
||||
* `configureGateway()` explicitly in their own beforeAll, which
|
||||
* overwrites this preload.
|
||||
*/
|
||||
import {
|
||||
configureGateway,
|
||||
getEmbeddingDimensions,
|
||||
__setGatewayResetBaselineForTests,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { configureGateway, getEmbeddingDimensions } from '../../src/core/ai/gateway.ts';
|
||||
import { beforeEach } from 'bun:test';
|
||||
|
||||
const LEGACY_CONFIG = {
|
||||
@@ -30,16 +26,12 @@ const LEGACY_CONFIG = {
|
||||
embedding_dimensions: 1536,
|
||||
} as const;
|
||||
|
||||
function legacyGatewayConfig() {
|
||||
return {
|
||||
function applyLegacy() {
|
||||
configureGateway({
|
||||
embedding_model: LEGACY_CONFIG.embedding_model,
|
||||
embedding_dimensions: LEGACY_CONFIG.embedding_dimensions,
|
||||
env: { ...process.env },
|
||||
};
|
||||
}
|
||||
|
||||
function applyLegacy() {
|
||||
configureGateway(legacyGatewayConfig());
|
||||
});
|
||||
}
|
||||
|
||||
if (process.env.GBRAIN_DEBUG_PRELOAD === '1') {
|
||||
@@ -49,16 +41,6 @@ if (process.env.GBRAIN_DEBUG_PRELOAD === '1') {
|
||||
// Initial application — covers tests that don't reset the gateway.
|
||||
applyLegacy();
|
||||
|
||||
// #3554: make resetGateway() mean "back to this baseline" instead of
|
||||
// "unconfigured". Without this, a file whose teardown calls resetGateway()
|
||||
// leaves _config = null; the NEXT file's beforeAll engine-connect then
|
||||
// reconfigures from the shipped default (zembed-1 @ 1280) BEFORE the
|
||||
// beforeEach below can fire, and the 1280-sized schema rejects the file's
|
||||
// 1536-d fixtures. Which file pairs collide depends on shard bin-packing,
|
||||
// so adding any test file reshuffles the mines. A factory (not a frozen
|
||||
// config) so each re-application captures fresh process.env.
|
||||
__setGatewayResetBaselineForTests(legacyGatewayConfig);
|
||||
|
||||
// Per-test re-application — handles tests that call `resetGateway()`
|
||||
// in their setup/teardown. Bun's preload allows registering global
|
||||
// hooks; this fires before every test in every file in the shard.
|
||||
|
||||
@@ -522,7 +522,7 @@ just content.
|
||||
const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true });
|
||||
expect(result.status).toBe('skipped');
|
||||
expect(result.error).toContain('no usable slug');
|
||||
expect(result.error).toContain('at least one letter or number (any script)');
|
||||
expect(result.error).toContain('ASCII / Chinese / Japanese / Korean');
|
||||
expect((engine as any)._calls.length).toBe(0);
|
||||
});
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
} from '../src/core/search/llm-intent.ts';
|
||||
import {
|
||||
__setChatTransportForTests,
|
||||
__unconfigureGatewayForTests,
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
@@ -123,9 +122,7 @@ describe('classifyModalityWithLLM — fail-open', () => {
|
||||
});
|
||||
|
||||
test('Gateway not configured → returns fallback', async () => {
|
||||
// Hard-unconfigure: resetGateway() would restore the preload's test
|
||||
// baseline (#3554), whose {...process.env} could make chat available.
|
||||
__unconfigureGatewayForTests();
|
||||
resetGateway();
|
||||
// No configureGateway called → isAvailable('chat') returns false.
|
||||
expect(await classifyModalityWithLLM('q', 'text')).toBe('text');
|
||||
});
|
||||
|
||||
@@ -282,19 +282,12 @@ describe('shell-audit: computeAuditFilename', () => {
|
||||
|
||||
describe('shell-audit: write', () => {
|
||||
let tmpDir: string;
|
||||
// #3554-sibling: the audit-dir preload sets GBRAIN_AUDIT_DIR once at
|
||||
// process start; deleting it here (instead of restoring) let every file
|
||||
// AFTER this one in the shard write audit fixtures to the operator's
|
||||
// real ~/.gbrain/audit/ — and failed audit-dir-preload.test.ts whenever
|
||||
// bin-packing placed it later in the shard. Restore the prior value.
|
||||
const priorAuditDir = process.env.GBRAIN_AUDIT_DIR;
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'shell-audit-test-'));
|
||||
process.env.GBRAIN_AUDIT_DIR = tmpDir;
|
||||
});
|
||||
afterAll(() => {
|
||||
if (priorAuditDir === undefined) delete process.env.GBRAIN_AUDIT_DIR;
|
||||
else process.env.GBRAIN_AUDIT_DIR = priorAuditDir;
|
||||
delete process.env.GBRAIN_AUDIT_DIR;
|
||||
});
|
||||
|
||||
test('GBRAIN_AUDIT_DIR env override resolves to the custom dir', () => {
|
||||
|
||||
@@ -354,15 +354,6 @@ describe('MinionQueue: #1737 per-handler default timeout', () => {
|
||||
expect(sub.timeout_ms).toBe(30 * 60 * 1000);
|
||||
});
|
||||
|
||||
// #3207 — facts-absorb is one LLM extraction call per page (same shape as
|
||||
// chronicle_extract) but was missing from HANDLER_DEFAULT_TIMEOUT_MS, so it
|
||||
// inherited the tight null-default wall-clock and was dead-lettered
|
||||
// mid-generation on slow chat providers (facts silently lost).
|
||||
test('facts-absorb gets the 10-min LLM-extraction default (#3207)', async () => {
|
||||
const job = await queue.add('facts-absorb', { slug: 'people/alice-example' });
|
||||
expect(job.timeout_ms).toBe(10 * 60 * 1000);
|
||||
});
|
||||
|
||||
test('contextual per-chunk reindex gets the 60-min default', async () => {
|
||||
const job = await queue.add('contextual_reindex_per_chunk', { page_slug: 'large-transcript' }, undefined, {
|
||||
allowProtectedSubmit: true,
|
||||
@@ -718,26 +709,6 @@ describe('MinionQueue: Prune', () => {
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() + 86400000) }); // future date = prune everything old enough
|
||||
expect(count).toBe(1); // only the cancelled one
|
||||
});
|
||||
|
||||
// #2712: --dry-run used to be silently ignored — the destructive default
|
||||
// ran and deleted rows while the operator believed they were previewing.
|
||||
test('dryRun counts prunable jobs without deleting', async () => {
|
||||
const job1 = await queue.add('sync', {});
|
||||
await queue.cancelJob(job1.id); // terminal → prunable
|
||||
|
||||
const wouldPrune = await queue.prune({ olderThan: new Date(Date.now() + 86400000), dryRun: true });
|
||||
expect(wouldPrune).toBe(1);
|
||||
|
||||
// The row must still exist after a dry run.
|
||||
const stillThere = await queue.getJob(job1.id);
|
||||
expect(stillThere).not.toBeNull();
|
||||
expect(stillThere!.status).toBe('cancelled');
|
||||
|
||||
// A real prune afterwards actually deletes it.
|
||||
const pruned = await queue.prune({ olderThan: new Date(Date.now() + 86400000) });
|
||||
expect(pruned).toBe(1);
|
||||
expect(await queue.getJob(job1.id)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// --- Stats (1 test) ---
|
||||
|
||||
@@ -19,23 +19,3 @@ describe('CLI_ONLY command reachability (#2900)', () => {
|
||||
expect(CLI_ONLY.has('reconcile-links')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// #3224 — same drift class: `backfill` has a full `case 'backfill'` handler
|
||||
// (cli.ts, dispatching to commands/backfill.ts) but was missing from CLI_ONLY,
|
||||
// so every invocation hit the generic "Unknown command" branch.
|
||||
describe('CLI_ONLY command reachability (#3224)', () => {
|
||||
test('`backfill` is in CLI_ONLY so dispatch reaches its handler', () => {
|
||||
expect(CLI_ONLY.has('backfill')).toBe(true);
|
||||
});
|
||||
|
||||
test('`gbrain backfill --help` is dispatched, not rejected as unknown', () => {
|
||||
const { spawnSync } = require('node:child_process') as typeof import('node:child_process');
|
||||
const result = spawnSync('bun', ['run', 'src/cli.ts', 'backfill', '--help'], {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, GBRAIN_HOME: '/tmp/gbrain-test-backfill-nonexistent' },
|
||||
});
|
||||
expect(result.stderr ?? '').not.toContain('Unknown command');
|
||||
expect(result.status).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,330 +0,0 @@
|
||||
import { afterEach, describe, expect, it, setDefaultTimeout } from 'bun:test';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { basename, join, resolve } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..', '..');
|
||||
const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json');
|
||||
const GUARD = resolve(REPO_ROOT, 'scripts', 'check-engine-dynamic-import.sh');
|
||||
const VERIFY_DISPATCHER = resolve(REPO_ROOT, 'scripts', 'run-verify-parallel.sh');
|
||||
const BASH = process.platform === 'win32'
|
||||
? resolve(process.env.ProgramFiles ?? 'C:\\Program Files', 'Git', 'bin', 'bash.exe')
|
||||
: 'bash';
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
setDefaultTimeout(30_000);
|
||||
|
||||
function fixture(name: string, content: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gbrain-engine-import-'));
|
||||
tempDirs.push(dir);
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, content, 'utf8');
|
||||
return path;
|
||||
}
|
||||
|
||||
function runGuard(files: string[] = [], cwd = REPO_ROOT) {
|
||||
const result = spawnSync(BASH, [GUARD, ...files], {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
return {
|
||||
code: result.status ?? -1,
|
||||
stdout: result.stdout ?? '',
|
||||
stderr: result.stderr ?? '',
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe('check-engine-dynamic-import.sh', () => {
|
||||
it('exists', () => {
|
||||
expect(existsSync(GUARD)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects and reports an unmarked dynamic import', () => {
|
||||
const path = fixture('violator.ts', "async function load() {\n return await import('./helper.ts');\n}\n");
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
expect(result.stderr).toContain("await import('./helper.ts')");
|
||||
});
|
||||
|
||||
it('allows a same-line marker for multiple non-gateway imports and ignores comment-only matches', () => {
|
||||
const path = fixture(
|
||||
'allowed.ts',
|
||||
[
|
||||
"// await import('./comment.ts')",
|
||||
'/*',
|
||||
" * await import('./block-body.ts')",
|
||||
' */',
|
||||
"const first = import('./first.ts'); const second = import('./second.ts'); // engine-dynamic-import-ok",
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.stdout).toContain('check-engine-dynamic-import: ok (1 file(s) scanned)');
|
||||
});
|
||||
|
||||
it('rejects live code after a closed leading block comment', () => {
|
||||
const path = fixture(
|
||||
'leading-block-comment.ts',
|
||||
"/* load only when needed */ const helper = await import('./helper.ts');\n",
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
expect(result.stderr).toContain("await import('./helper.ts')");
|
||||
});
|
||||
|
||||
it('ignores dynamic-import text wholly inside a multiline block comment', () => {
|
||||
const path = fixture(
|
||||
'multiline-block-comment.ts',
|
||||
[
|
||||
'/*',
|
||||
" * await import('./comment-only.ts')",
|
||||
' */',
|
||||
'const value = 1;',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.stdout).toContain('check-engine-dynamic-import: ok (1 file(s) scanned)');
|
||||
});
|
||||
|
||||
it('reports every violation across multiple files', () => {
|
||||
const first = fixture(
|
||||
'first-violator.ts',
|
||||
"const first = await import('./first.ts');\nconst second = await import('./second.ts');\n",
|
||||
);
|
||||
const second = fixture(
|
||||
'second-violator.ts',
|
||||
"/* explanation */ const third = await import('./third.ts');\n",
|
||||
);
|
||||
const result = runGuard([first, second]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(first)}:1:`);
|
||||
expect(result.stderr).toContain(`${basename(first)}:2:`);
|
||||
expect(result.stderr).toContain(`${basename(second)}:1:`);
|
||||
}, 30_000);
|
||||
|
||||
it('does not mistake comment delimiters inside literals for comments', () => {
|
||||
const path = fixture(
|
||||
'literal-delimiters.ts',
|
||||
[
|
||||
'const url = "https://example.test";',
|
||||
'const block = "/* not a comment";',
|
||||
'const template = `https://example.test`;',
|
||||
'const pattern = /\\/\\//;',
|
||||
"const helper = await import('./helper.ts');",
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:5:`);
|
||||
}, 30_000);
|
||||
|
||||
it('detects bare and trivia-separated dynamic imports', () => {
|
||||
const path = fixture(
|
||||
'dynamic-import-syntax.ts',
|
||||
[
|
||||
"const first = import('./first.ts');",
|
||||
"const second = await import /* explanation */ ('./second.ts');",
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
}, 30_000);
|
||||
|
||||
it('detects live code after a multiline block comment closes', () => {
|
||||
const path = fixture(
|
||||
'after-multiline-comment.ts',
|
||||
"/*\n * explanation\n */ const helper = await import('./helper.ts');\n",
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:3:`);
|
||||
});
|
||||
|
||||
it('requires the allow marker on the import line', () => {
|
||||
const path = fixture(
|
||||
'marker-line.ts',
|
||||
"// engine-dynamic-import-ok\nconst helper = await import('./helper.ts');\n",
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
});
|
||||
|
||||
it('does not accept marker text outside comment trivia or within longer comment tokens', () => {
|
||||
const path = fixture(
|
||||
'marker-text.ts',
|
||||
[
|
||||
"const first = import('./engine-dynamic-import-ok.ts');",
|
||||
"const marker = 'engine-dynamic-import-ok'; const second = import('./second.ts');",
|
||||
'const template = `prefix',
|
||||
'// engine-dynamic-import-ok ${import("./third.ts")}`;',
|
||||
"const fourth = import('./fourth.ts'); // no-engine-dynamic-import-ok: not approved",
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:4:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:5:`);
|
||||
});
|
||||
|
||||
it('does not accept markers adjacent to Unicode identifier characters', () => {
|
||||
const path = fixture(
|
||||
'unicode-marker-text.ts',
|
||||
[
|
||||
"const first = import('./first.ts'); // noéengine-dynamic-import-ok: not approved",
|
||||
"const second = import('./second.ts'); // engine-dynamic-import-oké: not approved",
|
||||
"const third = import('./third.ts'); // éengine-dynamic-import-oké: not approved",
|
||||
"const fourth = import('./fourth.ts'); // nóengine-dynamic-import-ok: not approved",
|
||||
"const fifth = import('./fifth.ts'); // 𐐀engine-dynamic-import-ok: not approved",
|
||||
"const sixth = import('./sixth.ts'); // engine-dynamic-import-ok𐐀: not approved",
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:3:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:4:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:5:`);
|
||||
expect(result.stderr).toContain(`${basename(path)}:6:`);
|
||||
});
|
||||
|
||||
it('allows a marker in real multiline comment trivia on the import line', () => {
|
||||
const path = fixture(
|
||||
'multiline-marker.ts',
|
||||
[
|
||||
'/* rationale',
|
||||
' * engine-dynamic-import-ok */ const helper = import("./helper.ts");',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(0);
|
||||
});
|
||||
|
||||
it('fails on TypeScript parse diagnostics', () => {
|
||||
const path = fixture('malformed.ts', 'const broken = ;\n');
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain('cannot parse input file');
|
||||
expect(result.stderr).toContain(basename(path));
|
||||
});
|
||||
|
||||
it('reports recovered-AST violations alongside parse diagnostics', () => {
|
||||
const path = fixture(
|
||||
'malformed-violator.ts',
|
||||
"const helper = import('./helper.ts');\nconst broken = ;\n",
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain('cannot parse input file');
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
});
|
||||
|
||||
it('ignores type-position imports', () => {
|
||||
const path = fixture(
|
||||
'type-import.ts',
|
||||
"type Helper = import('./helper.ts').Helper;\n",
|
||||
);
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(0);
|
||||
});
|
||||
|
||||
it('reports readable-file violations alongside missing inputs', () => {
|
||||
const path = fixture('mixed-violator.ts', "const helper = import('./helper.ts');\n");
|
||||
const missing = join(tmpdir(), `gbrain-engine-import-missing-${process.pid}.ts`);
|
||||
const result = runGuard([path, missing]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:1:`);
|
||||
expect(result.stderr).toContain('cannot read input file');
|
||||
expect(result.stderr).toContain(basename(missing));
|
||||
});
|
||||
|
||||
it('fails when an explicit input file is missing', () => {
|
||||
const missing = join(tmpdir(), `gbrain-engine-import-missing-${process.pid}.ts`);
|
||||
const result = runGuard([missing]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain('cannot read input file');
|
||||
expect(result.stderr).toContain(basename(missing));
|
||||
});
|
||||
|
||||
it('resolves default inputs from the guard repository', () => {
|
||||
const foreign = mkdtempSync(join(tmpdir(), 'gbrain-engine-import-foreign-'));
|
||||
tempDirs.push(foreign);
|
||||
const foreignCore = join(foreign, 'src', 'core');
|
||||
mkdirSync(foreignCore, { recursive: true });
|
||||
expect(spawnSync('git', ['init', '-q'], { cwd: foreign }).status).toBe(0);
|
||||
writeFileSync(
|
||||
join(foreignCore, 'pglite-engine.ts'),
|
||||
"const foreign = import('./foreign.ts');\n",
|
||||
'utf8',
|
||||
);
|
||||
for (const name of ['postgres-engine.ts', 'migrate.ts']) {
|
||||
writeFileSync(join(foreignCore, name), '', 'utf8');
|
||||
}
|
||||
|
||||
const result = runGuard([], foreign);
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.stdout).toContain('check-engine-dynamic-import: ok (3 file(s) scanned)');
|
||||
}, 30_000);
|
||||
|
||||
it('still catches a violation in CRLF input', () => {
|
||||
const path = fixture('crlf.ts', "async function load() {\r\n return await import('./helper.ts');\r\n}\r\n");
|
||||
const result = runGuard([path]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain(`${basename(path)}:2:`);
|
||||
});
|
||||
|
||||
it('passes on the reconciled repository sources', () => {
|
||||
const result = runGuard();
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.stdout).toContain('check-engine-dynamic-import: ok (3 file(s) scanned)');
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('engine dynamic-import guard wiring', () => {
|
||||
it('is invoked through bash by check:all', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf8')) as {
|
||||
scripts: Record<string, string>;
|
||||
};
|
||||
expect(pkg.scripts['check:engine-dynamic-import']).toBe(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
expect(pkg.scripts['check:all']).toContain(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
});
|
||||
|
||||
it('is listed by the authoritative verify dispatcher', () => {
|
||||
const result = spawnSync(BASH, [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(new Set((result.stdout ?? '').trim().split('\n'))).toContain(
|
||||
'check:engine-dynamic-import',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
|
||||
});
|
||||
|
||||
describe('KNOBS_HASH_VERSION', () => {
|
||||
it('is 14 (13→14 compiled_truth boost no longer applies at detail=medium, so pre-fix rankings must be unreachable, #3430)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(14);
|
||||
it('is 13 (12→13 embedding-provider migration invalidates rows written against the prior embedding space, #3390)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
/**
|
||||
* #3430: the compiled_truth boost must not apply at `detail=medium`.
|
||||
*
|
||||
* `COMPILED_TRUTH_BOOST = 2.0` is applied AFTER RRF score normalization. RRF's
|
||||
* entire dynamic range over a 100-deep pool is 1/60 → 1/160 (a factor of 2.67),
|
||||
* so a 2.0x multiplier consumes roughly three quarters of it. Break-even is
|
||||
* `2/(60+r) >= 1/60`, i.e. r <= 60 — so ANY boosted chunk in the first 60 ranks
|
||||
* outranks an unboosted rank-1 chunk. That is a categorical filter, not a tilt:
|
||||
* a page whose actual answer is in a `fenced_code` chunk returns the prose
|
||||
* chunk instead, and the code chunk leaves the result window entirely.
|
||||
*
|
||||
* The gate was written as `detail !== 'high'` — "high is special" — but the
|
||||
* documented contract in `src/core/operations.ts` is:
|
||||
*
|
||||
* low (compiled truth only), medium (default, all with dedup), high (all chunks)
|
||||
*
|
||||
* which makes LOW the special one. `low` already restricts to compiled_truth,
|
||||
* so a boost there is a no-op among equals; `medium` and `high` are both
|
||||
* supposed to see everything. Hence `detail === 'low'`.
|
||||
*
|
||||
* These tests pin the arithmetic, not the constant — they would still fail if
|
||||
* someone reintroduced a boost at medium with a different multiplier or behind
|
||||
* a score floor, which is why they assert final RANK rather than score.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { rrfFusion, RRF_K, shouldBoostCompiledTruth } from '../src/core/search/hybrid.ts';
|
||||
import { KNOBS_HASH_VERSION } from '../src/core/search/mode.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
function chunk(slug: string, chunkSource: string): SearchResult {
|
||||
return { slug, chunk_source: chunkSource, chunk_text: 'x', title: slug, score: 0 } as unknown as SearchResult;
|
||||
}
|
||||
|
||||
/** One vector arm: the correct answer at rank 0, then `n` compiled_truth chunks. */
|
||||
function poolWithAnswerFirst(n: number): SearchResult[] {
|
||||
const list = [chunk('code/answer', 'fenced_code')];
|
||||
for (let i = 0; i < n; i++) list.push(chunk(`prose/p${i}`, 'compiled_truth'));
|
||||
return list;
|
||||
}
|
||||
|
||||
function rankOfAnswer(results: SearchResult[]): number {
|
||||
return results.findIndex((r) => r.slug === 'code/answer');
|
||||
}
|
||||
|
||||
describe('#3430: the detail→boost mapping itself', () => {
|
||||
// These are the assertions that actually FAIL on master. The rrfFusion tests
|
||||
// below pin the arithmetic but pass either way, because they pass the boost
|
||||
// flag explicitly — they cannot see how hybridSearch decides it. This is the
|
||||
// wiring.
|
||||
test('ONLY detail=low boosts compiled_truth', () => {
|
||||
expect(shouldBoostCompiledTruth('low')).toBe(true);
|
||||
expect(shouldBoostCompiledTruth('medium')).toBe(false);
|
||||
expect(shouldBoostCompiledTruth('high')).toBe(false);
|
||||
});
|
||||
|
||||
test('an absent detail does not boost — medium is the documented default', () => {
|
||||
// Callers that omit detail get medium semantics, so the unset case must
|
||||
// match medium, not low. A `!== 'high'` spelling gets this backwards.
|
||||
expect(shouldBoostCompiledTruth(undefined)).toBe(false);
|
||||
expect(shouldBoostCompiledTruth(null)).toBe(false);
|
||||
});
|
||||
|
||||
test('an unrecognized detail value does not boost', () => {
|
||||
// Fail-open toward showing everything rather than silently filtering.
|
||||
expect(shouldBoostCompiledTruth('')).toBe(false);
|
||||
expect(shouldBoostCompiledTruth('LOW')).toBe(false);
|
||||
expect(shouldBoostCompiledTruth('detailed')).toBe(false);
|
||||
});
|
||||
|
||||
test('the cache version was bumped so pre-fix rankings are unreachable', () => {
|
||||
// Results are cached AFTER fusion, so rows written under the old boost
|
||||
// semantics would otherwise be served under the new ones for the whole TTL.
|
||||
// 13 was the pre-fix value.
|
||||
expect(KNOBS_HASH_VERSION).toBeGreaterThanOrEqual(14);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3430: compiled_truth boost scope', () => {
|
||||
test('boost OFF (detail=medium/high) keeps the vector-ranked answer at rank 0', () => {
|
||||
// The regression this file exists for. Pre-fix, medium passed applyBoost=true
|
||||
// and the answer landed at rank n — outside a 20-result window for n >= 20.
|
||||
for (const n of [10, 20, 40, 80]) {
|
||||
const fused = rrfFusion([poolWithAnswerFirst(n)], RRF_K, false);
|
||||
expect(rankOfAnswer(fused), `n=${n}: answer must stay first without the boost`).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('boost ON demonstrates the categorical displacement it causes', () => {
|
||||
// Documents WHY the boost cannot be on at medium. Not an endorsement of
|
||||
// these numbers — a characterization of the mechanism, so a future reader
|
||||
// sees the cost rather than re-deriving it.
|
||||
const observed = [10, 20, 40].map((n) => ({
|
||||
n,
|
||||
rank: rankOfAnswer(rrfFusion([poolWithAnswerFirst(n)], RRF_K, true)),
|
||||
}));
|
||||
// Displacement scales with pool composition: the answer is pushed back by
|
||||
// roughly one position per boosted chunk ahead of the break-even rank.
|
||||
for (const { n, rank } of observed) {
|
||||
expect(rank, `n=${n}: boosted chunks should displace the answer`).toBeGreaterThan(0);
|
||||
}
|
||||
// And past ~20 compiled_truth chunks it leaves a default-size window.
|
||||
expect(observed.find((o) => o.n === 20)!.rank).toBeGreaterThanOrEqual(20);
|
||||
});
|
||||
|
||||
test('with the boost off, compiled_truth still wins when the vector arm ranks it first', () => {
|
||||
// Guard against over-correcting: removing the boost must not penalize
|
||||
// compiled_truth, only stop privileging it.
|
||||
const list = [chunk('prose/answer', 'compiled_truth'), chunk('code/other', 'fenced_code')];
|
||||
const fused = rrfFusion([list], RRF_K, false);
|
||||
expect(fused[0].slug).toBe('prose/answer');
|
||||
});
|
||||
});
|
||||
@@ -413,10 +413,7 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
|
||||
// #3390/#3391: bumped 12→13 for the embedding-provider migration wave —
|
||||
// legacy callers hash prov=default before AND after a provider swap, so
|
||||
// pre-migration cache rows must become unreachable on upgrade.
|
||||
// v0.42.67.x bumped 13→14: the compiled_truth boost no longer applies at
|
||||
// detail=medium (#3430). Cached rows were ranked under the old semantics,
|
||||
// so they must become unreachable rather than be served under the new ones.
|
||||
expect(KNOBS_HASH_VERSION).toBe(14);
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
|
||||
@@ -581,8 +578,8 @@ describe('v0.40.4 — graph_signals knob', () => {
|
||||
});
|
||||
|
||||
describe('v0.42.3.0 — autocut knobs', () => {
|
||||
test('KNOBS_HASH_VERSION is 14 (13→14 compiled_truth boost scope fix, #3430)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(14);
|
||||
test('KNOBS_HASH_VERSION is 13 (12→13 embedding-migration wave, #3390/#3391)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
|
||||
|
||||
@@ -64,10 +64,7 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
// pre-fix document-side query vectors must not be served.
|
||||
// #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) —
|
||||
// cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes.
|
||||
// #3430: 13→14 — the compiled_truth boost no longer applies at
|
||||
// detail=medium. Results are cached after fusion, so rows ranked under
|
||||
// the old boost semantics must not be served under the new ones.
|
||||
expect(KNOBS_HASH_VERSION).toBe(14);
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('hash is 16 hex chars regardless of reranker config', () => {
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { EventEmitter } from 'events';
|
||||
import { waitForHttpServerLifecycle, type HttpServerLifecycle } from '../src/commands/serve-http.ts';
|
||||
|
||||
class FakeHttpServer extends EventEmitter {
|
||||
listening = true;
|
||||
closeCalls = 0;
|
||||
|
||||
close(callback?: (error?: Error) => void): this {
|
||||
this.closeCalls++;
|
||||
this.listening = false;
|
||||
queueMicrotask(() => {
|
||||
callback?.();
|
||||
this.emit('close');
|
||||
});
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
describe('HTTP server lifecycle', () => {
|
||||
test('waits for shared cleanup to close the server', async () => {
|
||||
const server = new FakeHttpServer();
|
||||
const signals = new EventEmitter();
|
||||
let cleanup: (() => Promise<void>) | undefined;
|
||||
let deregistered = false;
|
||||
let resolved = false;
|
||||
|
||||
const lifecycle = waitForHttpServerLifecycle(server as unknown as HttpServerLifecycle, {
|
||||
signals: signals as unknown as NodeJS.Process,
|
||||
register(_name, fn) {
|
||||
cleanup = fn;
|
||||
return () => { deregistered = true; };
|
||||
},
|
||||
}).then(() => { resolved = true; });
|
||||
|
||||
await Promise.resolve();
|
||||
expect(resolved).toBe(false);
|
||||
expect(cleanup).toBeDefined();
|
||||
|
||||
await cleanup!();
|
||||
await lifecycle;
|
||||
|
||||
expect(server.closeCalls).toBe(1);
|
||||
expect(deregistered).toBe(true);
|
||||
expect(signals.listenerCount('SIGINT')).toBe(0);
|
||||
});
|
||||
|
||||
test('SIGINT closes the server through the same idempotent path', async () => {
|
||||
const server = new FakeHttpServer();
|
||||
const signals = new EventEmitter();
|
||||
|
||||
const lifecycle = waitForHttpServerLifecycle(server as unknown as HttpServerLifecycle, {
|
||||
signals: signals as unknown as NodeJS.Process,
|
||||
register() {
|
||||
return () => {};
|
||||
},
|
||||
});
|
||||
|
||||
signals.emit('SIGINT');
|
||||
await lifecycle;
|
||||
|
||||
expect(server.closeCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -1,115 +0,0 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { slugifySegment, slugifyPath } from '../src/core/sync.ts';
|
||||
import { validatePageSlug } from '../src/core/operations.ts';
|
||||
import { isValidHolder } from '../src/core/takes-fence.ts';
|
||||
|
||||
/**
|
||||
* #3417 — silent data loss for non-Latin, non-CJK scripts.
|
||||
*
|
||||
* Pre-fix, slugifySegment stripped every character outside [a-z0-9._-] + CJK,
|
||||
* so whole filenames in Hebrew / Arabic / Cyrillic / Greek / Thai collapsed to
|
||||
* empty segments. Distinct files then mapped to the SAME slug (their shared
|
||||
* directory prefix) and last-writer-wins overwrote each other with `import`
|
||||
* reporting 0 errors.
|
||||
*
|
||||
* Every assertion here is behavioral (input → output), so this file FAILS on
|
||||
* pre-fix master and passes with the Unicode-property-escape grammar.
|
||||
*/
|
||||
|
||||
describe('#3417: non-Latin scripts survive slugification', () => {
|
||||
// The six script families from the issue, before/after.
|
||||
const cases: Array<[string, string, string]> = [
|
||||
['Hebrew', 'notes/רשימת קניות.md', 'notes/רשימת-קניות'],
|
||||
['Arabic', 'notes/قائمة المهام.md', 'notes/قائمة-المهام'],
|
||||
['Cyrillic', 'notes/Список задач.md', 'notes/список-задач'],
|
||||
// Greek: tonos marks decompose to U+0301 under NFD and are stripped by the
|
||||
// same combining-accent pass that turns café → cafe. Consistent, stable.
|
||||
['Greek', 'notes/Λίστα εργασιών.md', 'notes/λιστα-εργασιων'],
|
||||
['Thai', 'notes/รายการซื้อของ.md', 'notes/รายการซื้อของ'],
|
||||
['Hebrew + digits', 'notes/תוכנית עבודה 2026.md', 'notes/תוכנית-עבודה-2026'],
|
||||
];
|
||||
|
||||
for (const [name, input, expected] of cases) {
|
||||
test(`${name}: ${input} → ${expected}`, () => {
|
||||
expect(slugifyPath(input)).toBe(expected);
|
||||
});
|
||||
}
|
||||
|
||||
test('distinct same-directory files no longer collapse onto one slug', () => {
|
||||
// Pre-fix ALL of these slugified to "notes" — one page, last writer wins.
|
||||
const slugs = [
|
||||
slugifyPath('notes/רשימת קניות.md'),
|
||||
slugifyPath('notes/قائمة المهام.md'),
|
||||
slugifyPath('notes/Список задач.md'),
|
||||
slugifyPath('notes/Λίστα εργασιών.md'),
|
||||
slugifyPath('notes/รายการซื้อของ.md'),
|
||||
];
|
||||
expect(new Set(slugs).size).toBe(slugs.length);
|
||||
for (const s of slugs) expect(s).not.toBe('notes');
|
||||
});
|
||||
|
||||
test('emitted slugs are ACCEPTED by validatePageSlug (three-grammar coherence)', () => {
|
||||
// The trap: fixing only sync.ts makes sync emit slugs put_page rejects.
|
||||
for (const [, input] of cases) {
|
||||
const slug = slugifyPath(input);
|
||||
expect(() => validatePageSlug(slug)).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test('takes-fence holder grammar accepts non-Latin slugs', () => {
|
||||
expect(isValidHolder('people/גארי-כהן')).toBe(true);
|
||||
expect(isValidHolder('companies/شركة-مثال')).toBe(true);
|
||||
// Uppercase still rejected (lowercase-canonical contract preserved).
|
||||
expect(isValidHolder('people/Garry-Tan')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3417: normalization — NFD (macOS) and NFC (git/Linux) converge', () => {
|
||||
test('Hebrew NFD filename produces the same slug as NFC', () => {
|
||||
const nfc = 'notes/רשימת קניות.md'.normalize('NFC');
|
||||
const nfd = 'notes/רשימת קניות.md'.normalize('NFD');
|
||||
expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
|
||||
});
|
||||
|
||||
test('Vietnamese NFD filename produces the same slug as NFC', () => {
|
||||
const nfc = 'notes/người dùng.md'.normalize('NFC');
|
||||
const nfd = 'notes/người dùng.md'.normalize('NFD');
|
||||
expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3417: regressions — existing behavior unchanged', () => {
|
||||
test('ASCII kebab-casing, lowercasing, dots, underscores', () => {
|
||||
expect(slugifyPath('notes/Shopping List.md')).toBe('notes/shopping-list');
|
||||
expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0');
|
||||
expect(slugifySegment('my_file_name')).toBe('my_file_name');
|
||||
expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024');
|
||||
});
|
||||
|
||||
test('Latin accents still strip (café → cafe)', () => {
|
||||
expect(slugifySegment('café résumé')).toBe('cafe-resume');
|
||||
});
|
||||
|
||||
test('CJK still preserved', () => {
|
||||
expect(slugifyPath('notes/购物清单.md')).toBe('notes/购物清单');
|
||||
expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经');
|
||||
expect(slugifySegment('한글테스트'.normalize('NFD'))).toBe('한글테스트');
|
||||
});
|
||||
|
||||
test('all-symbol input still collapses to empty (frontmatter-fallback path intact)', () => {
|
||||
expect(slugifySegment('!!!')).toBe('');
|
||||
expect(slugifySegment('🎉🎉')).toBe('');
|
||||
});
|
||||
|
||||
test('control chars, RTL override, punctuation still stripped', () => {
|
||||
expect(slugifySegment('evilgnp')).toBe('evilgnp');
|
||||
expect(slugifySegment('a\u0000b')).toBe('ab');
|
||||
});
|
||||
|
||||
test('validatePageSlug still rejects traversal, backslash, RTL override, uppercase-only weirdness', () => {
|
||||
expect(() => validatePageSlug('../etc/passwd')).toThrow();
|
||||
expect(() => validatePageSlug('notes\\file')).toThrow();
|
||||
expect(() => validatePageSlug('notes/evil')).toThrow();
|
||||
expect(() => validatePageSlug('notes/a\u0007b')).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -259,10 +259,10 @@ describe('SLUG_SEGMENT_PATTERN (v0.32.7)', () => {
|
||||
expect(SLUG_SEGMENT_PATTERN.test('icp-理想客户画像')).toBe(true);
|
||||
});
|
||||
|
||||
test('accepts non-CJK Unicode (Vietnamese) since the #3417 all-script widening', () => {
|
||||
// Pre-#3417 this was rejected (scope was CJK only). The grammar now uses
|
||||
// Unicode property escapes, so đ/ư/etc. are valid slug characters.
|
||||
const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`, 'u'));
|
||||
expect(result).not.toBeNull();
|
||||
test('REGRESSION: rejects non-CJK Unicode (Vietnamese)', () => {
|
||||
// Scope is CJK only; Vietnamese with combining diacritics stays rejected
|
||||
// until we widen to Unicode property escapes in v0.33+.
|
||||
const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`));
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
/**
|
||||
* #3056 — sync rename path: a failed `updateSlug` must not leave a live
|
||||
* duplicate of the renamed page behind.
|
||||
*
|
||||
* Before the fix, the rename loop swallowed `updateSlug` failures with an
|
||||
* empty catch ("treat as add") and could not see a zero-row UPDATE at all
|
||||
* (updateSlug returned void). The run then fell through to importFile,
|
||||
* which created/updated the row at the new path — while the old row stayed
|
||||
* behind, live, with its slug occupied. Nothing was logged, no counter
|
||||
* moved, and the duplicate was permanent.
|
||||
*
|
||||
* The fix reconciles: when the cheap rename didn't move a row AND the
|
||||
* destination demonstrably materialized, the stale old row is located
|
||||
* positively by `source_path = from` and deleted. Two safety rails:
|
||||
*
|
||||
* - dedup-skip protection: identity dedup can skip the import against
|
||||
* the OLD row, in which case nothing landed at the destination and
|
||||
* deleting the old row would destroy the only copy — no reconcile.
|
||||
* - no slug-guess deletes: the stale row is found by source_path only;
|
||||
* an unrelated row that happens to sit at the guessed slug survives.
|
||||
*
|
||||
* A failed reconcile delete lands in failedFiles so the existing failure
|
||||
* gate blocks the bookmark and the next run retries the same rename diff.
|
||||
*/
|
||||
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
const repos: string[] = [];
|
||||
// Serial-file requirement: blocked runs write real rows to the sync-failure
|
||||
// ledger under the gbrain home — isolate it per test so the operator's
|
||||
// actual ledger is never touched (GBRAIN_HOME is the isolation lever;
|
||||
// process.env.HOME does not redirect Bun's os.homedir()).
|
||||
let tmpHome: string;
|
||||
const originalGbrainHome = process.env.GBRAIN_HOME;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-3056-home-'));
|
||||
process.env.GBRAIN_HOME = tmpHome;
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalGbrainHome !== undefined) process.env.GBRAIN_HOME = originalGbrainHome;
|
||||
else delete process.env.GBRAIN_HOME;
|
||||
try { rmSync(tmpHome, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
while (repos.length) {
|
||||
const d = repos.pop();
|
||||
if (d) rmSync(d, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function personMd(title: string, body: string): string {
|
||||
return ['---', 'type: person', `title: ${title}`, '---', '', body].join('\n');
|
||||
}
|
||||
|
||||
/** Create a temp git repo seeded with the given files + an initial commit. */
|
||||
function mkRepo(files: Record<string, string>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gbrain-3056-'));
|
||||
repos.push(dir);
|
||||
execSync('git init', { cwd: dir, stdio: 'pipe' });
|
||||
execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' });
|
||||
execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' });
|
||||
for (const [rel, content] of Object.entries(files)) {
|
||||
mkdirSync(join(dir, rel, '..'), { recursive: true });
|
||||
writeFileSync(join(dir, rel), content);
|
||||
}
|
||||
execSync('git add -A && git commit -m "initial"', { cwd: dir, stdio: 'pipe' });
|
||||
return dir;
|
||||
}
|
||||
|
||||
const SYNC_OPTS = { noPull: true, noEmbed: true, noExtract: true, sourceId: 'default' } as const;
|
||||
|
||||
async function countPages(): Promise<number> {
|
||||
const rows = await engine.executeRaw<{ n: number | string }>(
|
||||
`SELECT count(*)::int AS n FROM pages WHERE source_id = 'default'`,
|
||||
);
|
||||
return Number(rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
describe('updateSlug engine contract (#3056)', () => {
|
||||
test('returns 1 when the old slug row is moved', async () => {
|
||||
await engine.putPage('people/old', {
|
||||
type: 'person', title: 'Old', compiled_truth: 'body',
|
||||
}, { sourceId: 'default' });
|
||||
const moved = await engine.updateSlug('people/old', 'people/new', { sourceId: 'default' });
|
||||
expect(moved).toBe(1);
|
||||
expect(await engine.getPage('people/new')).not.toBeNull();
|
||||
});
|
||||
|
||||
test('returns 0 when the old slug has no row (the silent no-op case)', async () => {
|
||||
const moved = await engine.updateSlug('people/ghost', 'people/new', { sourceId: 'default' });
|
||||
expect(moved).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3056: rename fallback reconciles the stale old row', () => {
|
||||
test('collision: destination slug occupied → stale old row deleted after import lands', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const repo = mkRepo({ 'people/carol.md': personMd('Carol', 'Carol is a person.') });
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(await engine.getPage('people/carol')).not.toBeNull();
|
||||
|
||||
// A pre-existing row already occupies the rename destination, so
|
||||
// updateSlug throws (source_id, slug) UNIQUE and the loop falls back.
|
||||
await engine.putPage('people/dana', {
|
||||
type: 'person', title: 'Dana (stale)', compiled_truth: 'occupies the destination slug',
|
||||
}, { sourceId: 'default' });
|
||||
|
||||
execSync('git mv people/carol.md people/dana.md', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git commit -m "rename carol to dana"', { cwd: repo, stdio: 'pipe' });
|
||||
|
||||
const result = await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(result.status).toBe('synced');
|
||||
|
||||
// The destination carries the renamed file's content...
|
||||
const dana = await engine.getPage('people/dana');
|
||||
expect(dana).not.toBeNull();
|
||||
expect(dana!.compiled_truth).toContain('Carol is a person.');
|
||||
|
||||
// ...and the stale old row is gone — no live duplicate.
|
||||
expect(await engine.getPage('people/carol')).toBeNull();
|
||||
expect(await countPages()).toBe(1);
|
||||
});
|
||||
|
||||
test('dedup-skip against the old row must NOT reconcile: the only copy survives', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// frontmatter.id gives identity dedup a handle: the import at the new
|
||||
// path can skip as "identical to <old row>" — in which case NOTHING
|
||||
// landed at the destination and deleting the old row would destroy the
|
||||
// only copy of the content.
|
||||
const md = ['---', 'type: person', 'title: Carol', 'id: ext-3056', '---', '', 'Carol is a person.'].join('\n');
|
||||
const repo = mkRepo({ 'people/carol.md': md });
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(await engine.getPage('people/carol')).not.toBeNull();
|
||||
|
||||
// Destination occupied → updateSlug throws → fallback path.
|
||||
await engine.putPage('people/dana', {
|
||||
type: 'person', title: 'Dana (stale)', compiled_truth: 'occupies the destination slug',
|
||||
}, { sourceId: 'default' });
|
||||
|
||||
execSync('git mv people/carol.md people/dana.md', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git commit -m "rename carol to dana"', { cwd: repo, stdio: 'pipe' });
|
||||
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
|
||||
// The import skipped against the OLD row (identity dedup), so the
|
||||
// destination never materialized with the renamed content — the
|
||||
// reconcile must not have deleted the old row, which still holds the
|
||||
// only copy.
|
||||
const carol = await engine.getPage('people/carol');
|
||||
expect(carol).not.toBeNull();
|
||||
expect(carol!.compiled_truth).toContain('Carol is a person.');
|
||||
});
|
||||
|
||||
test('reconcile never deletes by slug guess: unrelated manual row survives', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const repo = mkRepo({ 'people/carol.md': personMd('Carol', 'Carol is a person.') });
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
|
||||
// The file's real row drifts to a divergent slug with no source_path
|
||||
// (unlocatable), and an UNRELATED manually-curated page happens to sit
|
||||
// at the path-derived slug a naive reconcile would guess.
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET slug = 'people/carol-divergent', source_path = NULL
|
||||
WHERE source_id = 'default' AND slug = 'people/carol'`,
|
||||
);
|
||||
await engine.putPage('people/carol', {
|
||||
type: 'person', title: 'Manual Carol', compiled_truth: 'hand-authored, not from the file',
|
||||
}, { sourceId: 'default' });
|
||||
// Destination occupied → updateSlug throws UNIQUE → fallback path.
|
||||
await engine.putPage('people/dana', {
|
||||
type: 'person', title: 'Dana (stale)', compiled_truth: 'occupies the destination slug',
|
||||
}, { sourceId: 'default' });
|
||||
|
||||
execSync('git mv people/carol.md people/dana.md', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git commit -m "rename carol to dana"', { cwd: repo, stdio: 'pipe' });
|
||||
|
||||
const result = await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(result.status).toBe('synced');
|
||||
|
||||
// The destination materialized with the file's content...
|
||||
const dana = await engine.getPage('people/dana');
|
||||
expect(dana).not.toBeNull();
|
||||
expect(dana!.compiled_truth).toContain('Carol is a person.');
|
||||
// ...but no row had source_path = from, so the reconcile deleted
|
||||
// NOTHING: the unrelated manual row at the guessed slug survives.
|
||||
const manual = await engine.getPage('people/carol');
|
||||
expect(manual).not.toBeNull();
|
||||
expect(manual!.compiled_truth).toContain('hand-authored');
|
||||
});
|
||||
|
||||
test('happy path: clean git mv rename keeps page_id and touches nothing else', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const repo = mkRepo({ 'people/carol.md': personMd('Carol', 'Carol is a person.') });
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
const before = await engine.getPage('people/carol');
|
||||
expect(before).not.toBeNull();
|
||||
|
||||
execSync('git mv people/carol.md people/dana.md', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git commit -m "rename carol to dana"', { cwd: repo, stdio: 'pipe' });
|
||||
|
||||
const result = await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(result.status).toBe('synced');
|
||||
|
||||
const after = await engine.getPage('people/dana');
|
||||
expect(after).not.toBeNull();
|
||||
expect(after!.id).toBe(before!.id); // cheap-path rename preserved the row
|
||||
expect(await engine.getPage('people/carol')).toBeNull();
|
||||
expect(await countPages()).toBe(1);
|
||||
});
|
||||
|
||||
test('reconcile failure blocks the bookmark and the next run retries to convergence', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const repo = mkRepo({ 'people/carol.md': personMd('Carol', 'Carol is a person.') });
|
||||
await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
await engine.putPage('people/dana', {
|
||||
type: 'person', title: 'Dana (stale)', compiled_truth: 'occupies the destination slug',
|
||||
}, { sourceId: 'default' });
|
||||
|
||||
execSync('git mv people/carol.md people/dana.md', { cwd: repo, stdio: 'pipe' });
|
||||
execSync('git commit -m "rename carol to dana"', { cwd: repo, stdio: 'pipe' });
|
||||
|
||||
// Inject a transient failure into the reconcile delete.
|
||||
const origDelete = engine.deletePage.bind(engine);
|
||||
engine.deletePage = async () => { throw new Error('injected transient delete failure'); };
|
||||
let blocked;
|
||||
try {
|
||||
blocked = await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
} finally {
|
||||
engine.deletePage = origDelete;
|
||||
}
|
||||
|
||||
// The failed reconcile is not checkpointed past: the run blocks and the
|
||||
// stale duplicate is still visible. The failure is recorded as a
|
||||
// `<rename:…>` SENTINEL, which the auto-skip valve can never
|
||||
// chronic-skip — an outage lasting longer than the threshold must not
|
||||
// quietly bank the duplicate.
|
||||
expect(blocked.status).toBe('blocked_by_failures');
|
||||
expect(blocked.failedFiles).toBe(1);
|
||||
expect(await engine.getPage('people/carol')).not.toBeNull();
|
||||
const { loadSyncFailures } = await import('../src/core/sync-failure-ledger.ts');
|
||||
const openSentinels = loadSyncFailures().filter(
|
||||
f => f.path === '<rename:people/dana.md>' && f.state === 'open',
|
||||
);
|
||||
expect(openSentinels).toHaveLength(1);
|
||||
|
||||
// Next run (failure gone) retries the same rename diff and converges.
|
||||
const result = await performSync(engine, { repoPath: repo, ...SYNC_OPTS });
|
||||
expect(result.status).toBe('synced');
|
||||
expect(await engine.getPage('people/carol')).toBeNull();
|
||||
const dana = await engine.getPage('people/dana');
|
||||
expect(dana).not.toBeNull();
|
||||
expect(dana!.compiled_truth).toContain('Carol is a person.');
|
||||
expect(await countPages()).toBe(1);
|
||||
|
||||
// The convergence also clears the sentinel row — doctor must not keep
|
||||
// warning about a rename that has since reconciled.
|
||||
const remaining = loadSyncFailures().filter(
|
||||
f => f.path === '<rename:people/dana.md>' && f.state === 'open',
|
||||
);
|
||||
expect(remaining).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,76 +0,0 @@
|
||||
/**
|
||||
* #2079 — `gbrain takes list` used to parse "list" as a PAGE SLUG: cmdList
|
||||
* looked up a page named "list" and printed "No takes on list." even when the
|
||||
* brain held many takes — reading exactly like an empty takes table, so
|
||||
* agents concluded there were no takes and moved on.
|
||||
*
|
||||
* Fix: `list` is a real subcommand (CLI parity with the takes_list op).
|
||||
* Bare `takes <slug>` still lists per-page.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runTakes } from '../src/commands/takes.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string> {
|
||||
const lines: string[] = [];
|
||||
const orig = console.log;
|
||||
console.log = (...args: unknown[]) => { lines.push(args.join(' ')); };
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
console.log = orig;
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
await engine.putPage('companies/acme-example', {
|
||||
type: 'company',
|
||||
title: 'Acme Example',
|
||||
compiled_truth: 'Acme Example is a test company.',
|
||||
});
|
||||
const [row] = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = 'companies/acme-example'`,
|
||||
);
|
||||
await engine.addTakesBatch([{
|
||||
page_id: row.id,
|
||||
row_num: 1,
|
||||
claim: 'Acme will ship the widget by Q3.',
|
||||
kind: 'bet',
|
||||
holder: 'self',
|
||||
weight: 0.7,
|
||||
}]);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
describe('gbrain takes list (#2079)', () => {
|
||||
test('`takes list` lists all takes instead of slug-ifying "list"', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['list']));
|
||||
expect(out).not.toContain('No takes on list.');
|
||||
expect(out).toContain('Acme will ship the widget by Q3.');
|
||||
expect(out).toContain('companies/acme-example');
|
||||
});
|
||||
|
||||
test('`takes list --json` returns the full take rows', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['list', '--json']));
|
||||
const parsed = JSON.parse(out);
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed.length).toBe(1);
|
||||
expect(parsed[0].claim).toContain('Acme will ship');
|
||||
});
|
||||
|
||||
test('per-page form still works: `takes <slug>`', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['companies/acme-example']));
|
||||
expect(out).toContain('# Takes on companies/acme-example');
|
||||
expect(out).toContain('Acme will ship the widget by Q3.');
|
||||
});
|
||||
});
|
||||
@@ -55,20 +55,24 @@ describe('v0.37 Lane A — defaults sweep', () => {
|
||||
test('A.5: embedding-column registry builtin defaults to ZE/1280 on empty config + gateway', async () => {
|
||||
// The registry's resolution chain is cfg > gateway > DEFAULT. With
|
||||
// no cfg AND no gateway, it should fall through to the canonical
|
||||
// default (ZE/1280). Hard-unconfigure first to exercise that path —
|
||||
// resetGateway() would restore the preload's 1536 baseline (#3554).
|
||||
const { __unconfigureGatewayForTests, resetGateway } = await import('../src/core/ai/gateway.ts');
|
||||
// default (ZE/1280). Reset gateway first to exercise that path.
|
||||
const { resetGateway } = await import('../src/core/ai/gateway.ts');
|
||||
const { getEmbeddingColumnRegistry } = await import('../src/core/search/embedding-column.ts');
|
||||
__unconfigureGatewayForTests();
|
||||
resetGateway();
|
||||
try {
|
||||
const reg = getEmbeddingColumnRegistry({ engine: 'pglite' } as any);
|
||||
expect(reg['embedding']).toBeDefined();
|
||||
expect(reg['embedding'].provider).toBe('zeroentropyai:zembed-1');
|
||||
expect(reg['embedding'].dimensions).toBe(1280);
|
||||
} finally {
|
||||
// Restore the preload's legacy baseline so the rest of the file's
|
||||
// tests (and subsequent files in this shard) see a configured gateway.
|
||||
resetGateway();
|
||||
// Re-apply legacy preload defaults so the rest of the file's tests
|
||||
// (and subsequent files in this shard) see a configured gateway.
|
||||
const { configureGateway } = await import('../src/core/ai/gateway.ts');
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user