mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 18:02:30 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14d2689ec8 | ||
|
|
b022b17484 | ||
|
|
2a5dd27d68 | ||
|
|
569e431e80 | ||
|
|
3e8d1ea6f4 | ||
|
|
705a93e490 |
@@ -1,16 +0,0 @@
|
||||
# Line-ending policy.
|
||||
#
|
||||
# Shell scripts MUST be checked out with LF endings on every platform.
|
||||
# Git for Windows installs with `core.autocrlf=true` by default, which
|
||||
# rewrites LF -> CRLF on checkout. A strict bash (WSL, Linux CI, macOS)
|
||||
# then chokes on the trailing CR:
|
||||
#
|
||||
# scripts/run-unit-parallel.sh: line 23: $'\r': command not found
|
||||
# scripts/run-unit-parallel.sh: line 24: set: pipefail : invalid option name
|
||||
# scripts/run-unit-parallel.sh: line 32: syntax error near unexpected token `$'{\r''
|
||||
#
|
||||
# That silently disabled `bun run test`, `bun run verify`, `bun run ci:local`
|
||||
# and `bun run test:e2e` for Windows contributors, since all four dispatch
|
||||
# through bash. `eol=lf` pins the checkout regardless of the user's
|
||||
# core.autocrlf setting.
|
||||
*.sh text eol=lf
|
||||
@@ -61,10 +61,7 @@ jobs:
|
||||
- name: Run JSONB double-encode parity tests on real Postgres
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
# --timeout also raises bun's 5s default hook budget (beforeAll/afterAll
|
||||
# do NOT inherit a test's third-arg timeout; verified on bun 1.3.x).
|
||||
# Every runner script in scripts/ passes it; bare invocations must too.
|
||||
run: bun test --timeout=60000 test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
run: bun test test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
|
||||
tier1:
|
||||
name: Tier 1 (Mechanical)
|
||||
@@ -91,7 +88,7 @@ jobs:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- name: Run Tier 1 E2E tests
|
||||
run: bun test --timeout=60000 test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
run: bun test test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
|
||||
@@ -158,7 +155,7 @@ jobs:
|
||||
}
|
||||
EOF
|
||||
- name: Run Tier 2 skill tests
|
||||
run: bun test --timeout=60000 test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
run: bun test test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
@@ -29,9 +29,7 @@ jobs:
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
# --timeout matches every scripts/ runner and covers hook budgets too
|
||||
# (bunfig.toml's timeout key is ignored by bun; hooks default to 5s).
|
||||
- run: bun test --timeout=60000
|
||||
- run: bun test
|
||||
- run: bun run verify
|
||||
- run: bun build --compile --target=${{ matrix.target }} --outfile bin/${{ matrix.artifact }} src/cli.ts
|
||||
- name: Attest build provenance
|
||||
|
||||
@@ -113,11 +113,6 @@ jobs:
|
||||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||||
- run: bun install
|
||||
- run: bun run verify
|
||||
# Guard: no bare `bun test` in workflows/scripts — bun ignores
|
||||
# bunfig.toml's timeout, and hooks (beforeAll/afterAll) get the 5s
|
||||
# default regardless of per-test third-arg timeouts. Runs directly
|
||||
# (not via verify's CHECKS array) to avoid a package.json edit.
|
||||
- run: bash scripts/check-bun-test-timeout.sh
|
||||
|
||||
serial-tests:
|
||||
# *.serial.test.ts at --max-concurrency=1. Lives in its own runner so
|
||||
|
||||
+2
-5
@@ -1,7 +1,4 @@
|
||||
# No trailing slash: a bare `node_modules/` pattern matches directories only,
|
||||
# so a *symlink* named node_modules slips past it and can be committed
|
||||
# (that's how the /tmp-pointing symlink in faf5cdba got in). Match any type.
|
||||
node_modules
|
||||
node_modules/
|
||||
bin/
|
||||
.DS_Store
|
||||
*.log
|
||||
@@ -18,7 +15,7 @@ supabase/.temp/
|
||||
# self-contained binaries (the bun --compile path embeds it via
|
||||
# `import path from 'admin/dist/index.html' with { type: 'file' }`).
|
||||
# Build via: cd admin && bun install && bun run build.
|
||||
admin/node_modules
|
||||
admin/node_modules/
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
|
||||
@@ -2,60 +2,6 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.42.68.1] - 2026-07-30
|
||||
|
||||
**If you run `gbrain reindex-frontmatter` or `gbrain backfill` on the default embedded database, they now work. Until this release both failed every time, after waiting 30 seconds.**
|
||||
|
||||
The embedded database allows one process at a time, and holds a lock to enforce it. These two commands opened a second connection to the same database from inside the process that already held that lock, then waited for a lock that could never be released — because the thing holding it was the waiting process itself. The wait ran its full 30 seconds and the command exited with an error naming a blocking process that was, in fact, itself. Both commands now reuse the connection that is already open.
|
||||
|
||||
Nothing changes for brains on Postgres, where a second connection was always allowed.
|
||||
|
||||
## To take advantage of v0.42.68.1
|
||||
|
||||
Nothing to undo — the commands failed without writing anything. Just run whichever you needed:
|
||||
```bash
|
||||
gbrain reindex-frontmatter
|
||||
```
|
||||
|
||||
## [0.42.67.0] - 2026-07-28
|
||||
|
||||
**If you develop GBrain on Windows, the test and check commands now actually run. Until this release they were quietly doing almost nothing.**
|
||||
|
||||
`bun run test`, `bun run verify`, `bun run ci:local` and `bun run test:e2e` all hand off to shell scripts, and on Windows that hand-off was broken in two separate places. The commands did not stop with an obvious error. They reported a result, so a run could look finished when barely any of the checks had actually inspected anything. On a clean Windows clone, `bun run verify` got 1 check to pass and 31 to fail. It now gets 25 to pass and 7 to fail, and none of the 7 are caused by this change.
|
||||
|
||||
The first problem was line endings. Git for Windows installs with `core.autocrlf=true`, which rewrites shell scripts to Windows line endings when you clone or check out. Bash refuses to run those, so a script died on its second line before doing any work. The scripts stored in the repository were always correct; only the copy on your disk was wrong. A new `.gitattributes` pins every `.sh` file to Unix line endings at checkout, no matter how your Git is configured.
|
||||
|
||||
The second problem was how the checks were started. Thirty three of them pointed straight at a `.sh` file. On macOS and Linux the shell reads the `#!/usr/bin/env bash` line at the top of the script and runs it correctly. Bun on Windows does not do that, so those commands failed the moment they were called. They now go through `bash` explicitly, the same way the other eleven were already written.
|
||||
|
||||
Nothing changes for macOS and Linux. No stored file content moves, and no check behaves differently on those platforms.
|
||||
|
||||
## To take advantage of v0.42.67.0
|
||||
|
||||
Only Windows contributors need to do anything, and only once. `.gitattributes` applies at checkout time, so shell scripts already sitting on your disk keep their old line endings until you refresh them.
|
||||
|
||||
1. **Refresh the working copy** from the repository root:
|
||||
```bash
|
||||
git rm --cached -r . -q
|
||||
git reset --hard
|
||||
```
|
||||
2. **Confirm bash can read the scripts:**
|
||||
```bash
|
||||
bash -n scripts/run-unit-parallel.sh
|
||||
```
|
||||
Silence means it worked. `$'\r': command not found` means step 1 did not take effect.
|
||||
3. **Run the gate:**
|
||||
```bash
|
||||
bun run verify
|
||||
```
|
||||
|
||||
### Itemized changes
|
||||
|
||||
- New root `.gitattributes` pins `*.sh text eol=lf`, so shell scripts check out with Unix line endings regardless of the contributor's `core.autocrlf` setting. All 59 tracked `.sh` files were already stored with Unix endings, so `git add --renormalize .` reports nothing to do and no stored content changes.
|
||||
- `package.json` now routes the remaining 33 `.sh` check commands through `bash`, matching the 11 that already did. Every tracked `.sh` file carries a bash shebang (52 `#!/usr/bin/env bash` and 7 `#!/bin/bash`), so the treatment is uniform across all of them.
|
||||
- The five `scripts/*.ts` entries still run under bun and are untouched.
|
||||
- `CONTRIBUTING.md` gains a Windows section covering the one-time working-copy refresh and the `bash scripts/<name>.sh` convention for new checks.
|
||||
- `docs/TESTING.md` records how the test commands dispatch through bash, and notes that three tree-walking checks plus `typecheck` can exceed the 120s per-check cap on Windows while passing on Linux and macOS.
|
||||
|
||||
## [0.42.66.1] - 2026-07-27
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -67,19 +67,6 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
|
||||
@@ -11,28 +11,6 @@ bun test
|
||||
|
||||
Requires Bun 1.0+.
|
||||
|
||||
### Windows
|
||||
|
||||
`bun run test`, `verify`, `ci:local` and `test:e2e` all dispatch through bash, so
|
||||
the shell scripts under `scripts/` must be checked out with Unix line endings.
|
||||
The root `.gitattributes` pins `*.sh text eol=lf`, which overrides the
|
||||
`core.autocrlf=true` that Git for Windows installs by default. A fresh clone is
|
||||
correct with no extra steps.
|
||||
|
||||
If you cloned before that pin existed, your working copy still has the old
|
||||
Windows line endings and bash will fail with `$'\r': command not found`. Refresh
|
||||
it once, from the repository root:
|
||||
|
||||
```bash
|
||||
git rm --cached -r . -q
|
||||
git reset --hard
|
||||
bash -n scripts/run-unit-parallel.sh # silence means bash can read the scripts
|
||||
```
|
||||
|
||||
Every `check:*` entry in `package.json` invokes its script as `bash scripts/<name>.sh`
|
||||
rather than relying on the shebang, because bun on Windows cannot exec a `.sh`
|
||||
directly. Keep that prefix when you add a new shell-script check.
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
|
||||
@@ -16,13 +16,6 @@ If you fetched this file by URL without cloning yet, the companion files live at
|
||||
|
||||
## Step 1: Install GBrain
|
||||
|
||||
> **NEVER install from the npm registry.** GBrain is not distributed on npm; the npm
|
||||
> package named `gbrain` is an unrelated package. Do NOT run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` (note the missing `github:` prefix — that's the trap). The only
|
||||
> supported sources are `github:garrytan/gbrain` and a git clone, exactly as shown below.
|
||||
> If an unrelated npm install is already present, remove it first
|
||||
> (`npm uninstall -g gbrain` / `bun remove -g gbrain`); `gbrain doctor` also detects this.
|
||||
|
||||
Default path (Bun is required — gbrain is a Bun + TypeScript runtime):
|
||||
|
||||
```bash
|
||||
|
||||
@@ -65,16 +65,6 @@ This is the difference between a search engine and a brain. Search finds the pag
|
||||
|
||||
## Install
|
||||
|
||||
> [!WARNING]
|
||||
> **GBrain is NOT distributed on npm.** The npm package named `gbrain` is an unrelated
|
||||
> package with no connection to this project. Do not run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` — you'll get something else, and it can shadow the real binary on
|
||||
> your PATH. Install and upgrade ONLY via the documented paths below
|
||||
> (`bun install -g github:garrytan/gbrain`, or `git clone` + `bun install && bun link`).
|
||||
> If you already ran the npm install by mistake: `npm uninstall -g gbrain` /
|
||||
> `bun remove -g gbrain`, then reinstall from GitHub. `gbrain doctor` detects a
|
||||
> shadowing npm install and prints the fix.
|
||||
|
||||
GBrain is designed to be installed and operated by an AI agent. The fastest path is to have your agent do it for you. The CLI and MCP paths below are for people who want to wire it up themselves.
|
||||
|
||||
### Have your agent install it (recommended)
|
||||
|
||||
@@ -1,25 +1,5 @@
|
||||
# TODOS
|
||||
|
||||
## v0.42.67.0 follow-ups (Windows build tooling)
|
||||
|
||||
Filed as follow-ups from v0.42.67.0 (`.gitattributes` LF pin for `*.sh` +
|
||||
`bash` prefix on the 33 `package.json` check commands). Both items are newly
|
||||
observable: before that release these checks never executed on Windows at all,
|
||||
so nothing about their runtime was measurable.
|
||||
|
||||
- [ ] **P2 — three guard scripts exceed the 120s `run-verify-parallel.sh` cap on Windows.**
|
||||
With the dispatch fixed, `bun run verify` on Windows gets 25 passes and 7 failures, and
|
||||
`check:privacy`, `check:test-names` and `check:test-isolation` are timeouts rather than
|
||||
real failures (they pass on Linux and macOS well inside the cap). They walk the tree with
|
||||
per-file shell loops, which is far slower under Windows process creation. Either raise the
|
||||
cap for these three, or replace the per-file loop with a single `grep -r` pass. Same cap
|
||||
swallows `typecheck`, though standalone `bun run typecheck` exits 0.
|
||||
- [ ] **P3 — `check:wasm` cannot create its `node_modules` symlink on Windows.**
|
||||
`scripts/check-wasm-embedded.sh` fails with `ln: failed to create symbolic link
|
||||
'/tmp/gbrain-wasm-check.XXXX/node_modules': No such file or directory`. Unprivileged
|
||||
Windows accounts cannot create symlinks without developer mode. Consider a junction, a
|
||||
copy, or skipping the check with a clear message when symlink creation is unavailable.
|
||||
|
||||
## community fix-wave follow-ups (filed v0.42.60.0)
|
||||
|
||||
- [x] **P2 — cherry-pick #2112's uncovered doctor.ts hunk.** Fix-wave A (#2820) superseded
|
||||
@@ -82,20 +62,17 @@ Deferred from the provider-agnostic plumbing wave (#1249/#1250/#1292/#2271/#2209
|
||||
Plan + review trail at `~/.claude/plans/system-instruction-you-are-working-keen-newell.md`.
|
||||
The eng-review + Codex outside-voice narrowed the wave to these deferrals:
|
||||
|
||||
- [x] **P2 — Capability-aware query expansion on OpenAI-compat providers (#2372).**
|
||||
- [ ] **P2 — Capability-aware query expansion on OpenAI-compat providers (#2372).**
|
||||
Expansion only runs for recipes that declare an `expansion` touchpoint, and only the
|
||||
native providers (anthropic/openai/google) do. To make expansion work on
|
||||
litellm/openrouter/groq/together/deepseek you must ADD expansion touchpoints to those
|
||||
chat-capable recipes AND add a `generateObject`→`generateText` capability fallback for
|
||||
backends without strict structured outputs. Feature-shaped; overlaps the general
|
||||
OpenAI-compat proxy story (`docs/designs/COMMUNITY_IDEAS.md`). Community PR #2373 is a
|
||||
starting point. Implemented by #2373 plus the DeepSeek/Groq/Together recipe wave,
|
||||
LiteLLM chat/expansion support, and the OpenRouter expansion touchpoint. Where:
|
||||
`src/core/ai/gateway.ts:expand`, recipe files, `types.ts` (ExpansionTouchpoint).
|
||||
- [x] **P2 — LiteLLM as a chat/expansion backend.** `litellm-proxy` declares ONLY an
|
||||
starting point. Where: `src/core/ai/gateway.ts:expand`, recipe files, `types.ts` (ExpansionTouchpoint).
|
||||
- [ ] **P2 — LiteLLM as a chat/expansion backend.** `litellm-proxy` declares ONLY an
|
||||
embedding touchpoint, so `think`/chat on LiteLLM is dead. Add chat (and expansion) so a
|
||||
LiteLLM proxy is a full LLM backend, not embedding-only. Implemented by #2208.
|
||||
The general OpenAI-compat proxy story.
|
||||
LiteLLM proxy is a full LLM backend, not embedding-only. The general OpenAI-compat proxy story.
|
||||
- [ ] **P3 — Per-model embedding dims metadata on `EmbeddingTouchpoint`.** `default_dims`
|
||||
is recipe-wide, so a recipe (ollama) can't carry different native dims per model. This
|
||||
wave added the modern ollama model NAMES + a `trust_custom_dims` passthrough (user supplies
|
||||
|
||||
@@ -19,29 +19,6 @@ Seven test command tiers, each with a clear scope:
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential. | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | The historical pre-check scripts (22, chained sequentially in package.json). Overlaps `verify` heavily but is NOT a superset — `verify`'s `CHECKS` array in `scripts/run-verify-parallel.sh` (~30 entries incl. typecheck) is the authoritative gate; `check:all` keeps a few local-only extras (trailing-newline, exports-count, no-legacy-getconnection). | ~10s | Local-only sweep for the extras. |
|
||||
|
||||
### Shell dispatch and Windows
|
||||
|
||||
All four of `test`, `verify`, `ci:local` and `test:e2e` hand off to shell scripts
|
||||
under `scripts/`, so every `check:*` entry in `package.json` invokes its script as
|
||||
`bash scripts/<name>.sh` instead of relying on the shebang — bun on Windows cannot
|
||||
exec a `.sh` directly. Add a new shell-script check with that same prefix. The
|
||||
`scripts/*.ts` entries run under bun and take no prefix.
|
||||
|
||||
The scripts must also be on disk with Unix line endings. A strict bash (WSL, Linux
|
||||
CI, macOS) rejects CRLF and dies on the script's first meaningful line; the Cygwin
|
||||
bash that ships with Git for Windows tolerates it, so a green local run is not by
|
||||
itself evidence that a script is CRLF-clean.
|
||||
The root `.gitattributes` pins `*.sh text eol=lf`, which overrides the
|
||||
`core.autocrlf=true` default that Git for Windows installs. Working copies cloned
|
||||
before that pin need a one-time `git rm --cached -r . -q && git reset --hard` to
|
||||
pick it up; see the Windows section of `CONTRIBUTING.md`.
|
||||
|
||||
Wallclock figures in the table above are from a Mac dev box. Windows is
|
||||
substantially slower because each check pays full process-creation cost, and three
|
||||
tree-walking checks (`check:privacy`, `check:test-names`, `check:test-isolation`)
|
||||
plus `typecheck` can exceed the 120s per-check cap in `run-verify-parallel.sh`
|
||||
there even though they pass on Linux and macOS.
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` across 10 matrix shards partitioned by weight-aware LPT bin-packing (`scripts/sharding.ts`) and INCLUDES `*.slow.test.ts` (the two outlier slow files run as dedicated jobs alongside the matrix). CI EXCLUDES `*.serial.test.ts` from the shards and runs them in a dedicated job via `bun run test:serial`, one bun process per file — keeping serial files out of the shard processes is what preserves the `mock.module` quarantine (a top-level mock in one file leaks into every other file sharing its process). `bun run verify` gets its own job too. CI is the ground truth for "did everything pass."
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -87,15 +87,6 @@ embedding proximity. Four layers, added after the incident in
|
||||
deciding "is this page already here, safe to NOT write a duplicate?" keys off
|
||||
`create_safety`, not a raw blended score.
|
||||
|
||||
**Extraction quarantine lane (issue #160):** pages carrying the unverified
|
||||
auto-extracted markers (frontmatter `provenance: auto-extracted` +
|
||||
`status: unverified`, see `src/core/extraction-review.ts`) rank as ordinary
|
||||
content — they are skipped by the compiled-truth fusion boost and by the
|
||||
`people/`/`companies/` namespace source-boost, and every search result from
|
||||
such a page carries `unverified: true` so agents can label the provenance.
|
||||
Promote or reject them via `gbrain extraction-pending` / `gbrain
|
||||
extraction-review`.
|
||||
|
||||
The `search` MCP/CLI op is **cheap-hybrid** (vector + keyword + RRF + pool +
|
||||
title + alias, expansion off); `query` is the full-control variant. NamedThingBench
|
||||
(`gbrain eval retrieval-quality`) gates these families on every PR. Diagnose a
|
||||
|
||||
@@ -31,7 +31,7 @@ receipt file from disk and re-renders it. The other modes need the brain.
|
||||
| `--budget-usd N` | unset | Abort before next call's projected cost would exceed cap. Models without a `pricing.ts` entry fail loud (codex #4). |
|
||||
| `--source db|fs` | `db` | `fs` is reserved for v0.33+. |
|
||||
| `--slug-prefix P` | unset | Filter takes to pages whose slug starts with P. |
|
||||
| `--models a,b,c` | `openai:gpt-5.2,anthropic:claude-opus-4-7,google:gemini-2.0-flash` | Comma-separated panel. |
|
||||
| `--models a,b,c` | `openai:gpt-4o,anthropic:claude-opus-4-7,google:gemini-1.5-pro` | Comma-separated panel. |
|
||||
| `--json` | off | Emit the full receipt to stdout. |
|
||||
|
||||
## Receipt JSON shape (`schema_version: 1`)
|
||||
@@ -50,7 +50,7 @@ receipt file from disk and re-renders it. The other modes need the brain.
|
||||
},
|
||||
"prompt_sha8": "abcd1234",
|
||||
"models_sha8": "abcd1234",
|
||||
"models": ["openai:gpt-5.2", "anthropic:claude-opus-4-7", "google:gemini-2.0-flash"],
|
||||
"models": ["openai:gpt-4o", "anthropic:claude-opus-4-7", "google:gemini-1.5-pro"],
|
||||
"cycles_run": 3,
|
||||
"successes_per_cycle": [3, 3, 2],
|
||||
"verdict": "pass",
|
||||
|
||||
@@ -59,12 +59,6 @@ streaming progress to stderr. It is idempotent: re-running with the same
|
||||
language produces identical vectors. `--json` prints a machine-readable
|
||||
result envelope but still requires `--yes` (or an interactive confirm).
|
||||
|
||||
No cache purge is needed. The resolved language is part of the query-cache
|
||||
key, so rows written under the previous language are unreachable after the
|
||||
switch — searches read the retokenized index immediately instead of being
|
||||
served pre-switch results for up to `search.cache.ttl_seconds`. Switching
|
||||
back reaches the original rows rather than rebuilding them.
|
||||
|
||||
## Recipe: accent-insensitive Portuguese (`pt_br`)
|
||||
|
||||
Brazilian Portuguese content often mixes accented and unaccented spellings
|
||||
|
||||
@@ -1,690 +0,0 @@
|
||||
# Engine Dynamic-Import Reconciliation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Reconstruct the missing engine-path static-import hardening, preserve the four load-bearing lazy gateway fallbacks, and prevent unreviewed dynamic imports from returning.
|
||||
|
||||
**Architecture:** Make the 13 safe engine/migration import statements static and leave only four line-marked `ai/gateway.ts` imports inside their existing soft-failure `try/catch` boundaries. Enforce that current state with a repository-anchored Bash wrapper delegating to a fail-closed TypeScript AST scanner, a hermetic Bun regression test, package/verify wiring, and current-state architecture documentation.
|
||||
|
||||
**Tech Stack:** TypeScript compiler API, Bun test runner, Bash, Git, generated llms documentation bundles.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Reconstruct directly on branch `claude/kind-meitner-330c90`, based on investigated `origin/master` commit `6136e139972a5449630b4f47f5ed7b4cbe5b811b` plus design commit `d7f52d8c`.
|
||||
- Do not merge or cherry-pick `48ada48f`, `248bfe55`, `ef4cf7a8`, or either historical branch wholesale.
|
||||
- Do not modify `VERSION`, `CHANGELOG.md`, `TODOS.md`, or release metadata; this is a no-version-bump reconciliation.
|
||||
- Keep all four `await import('./ai/gateway.ts')` calls lazy: PGLite and Postgres `initSchema`, plus both `_upsertChunksOnce` methods.
|
||||
- Every allowed lazy gateway line must carry `engine-dynamic-import-ok`; there is no file-level exemption.
|
||||
- Preserve the stronger gateway rationale: the static closure is large, and eager module evaluation would occur outside the local `try/catch`, potentially converting a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
- Describe the hoists as engine-path hardening. Do not claim every dynamic import deterministically causes a Windows crash; system-wide commit exhaustion confounded prior measurements.
|
||||
- Keep shared PGLite/Postgres behavior in parity.
|
||||
- Invoke repository shell scripts through `bash` in `package.json`.
|
||||
- Capture complete test/check output to workspace-local `.context/*.txt` files before inspecting it; never pipe a test command directly through `head` or `tail`.
|
||||
- Use `git log -G`, not `git log -S`, for any additional dynamic-to-static import history work.
|
||||
- Keep every implementation and verification commit local. Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after local completion.
|
||||
- Before editing any affected function, run GBrain `code_blast` and `code_callers` for that symbol and inspect any disambiguation candidates.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
- Create `scripts/check-engine-dynamic-import.sh` — repository-anchored Bash wrapper for default and explicit input routing.
|
||||
- Create `scripts/check-engine-dynamic-import.ts` — TypeScript AST policy scanner for runtime `import()` expressions, parse/read failures, and exact-line comment-trivia opt-outs.
|
||||
- Create `test/scripts/check-engine-dynamic-import.test.ts` — 22 hermetic adversarial, CRLF, fail-closed, real-tree, and wiring tests.
|
||||
- Modify `src/core/pglite-engine.ts` — hoist three safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/postgres-engine.ts` — hoist eight safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/migrate.ts` — hoist two safe migration helper import statements.
|
||||
- Modify `package.json` — expose `check:engine-dynamic-import` and append it to `check:all` through `bash`.
|
||||
- Modify `scripts/run-verify-parallel.sh` — add the package check to the authoritative verify dispatcher.
|
||||
- Modify `CLAUDE.md` — add the cross-cutting current-state invariant.
|
||||
- Modify `docs/architecture/KEY_FILES.md` — update current-state entries for the three engine-path files.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` — required derived bundles after CLAUDE/reference documentation changes.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Establish and enforce the source invariant
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/check-engine-dynamic-import.sh`
|
||||
- Create: `scripts/check-engine-dynamic-import.ts`
|
||||
- Create: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `src/core/pglite-engine.ts`
|
||||
- Modify: `src/core/postgres-engine.ts`
|
||||
- Modify: `src/core/migrate.ts`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: shell positional arguments `FILE...`; without arguments, the guard scans the three repository files.
|
||||
- Produces: `scripts/check-engine-dynamic-import.sh [FILE...]`, exit `0` when every runtime dynamic import is allowed and exit `1` after reporting every `file:line:text` violation plus every read/parse error on stderr.
|
||||
- Produces: one line-level opt-out token, `engine-dynamic-import-ok`, accepted only in real comment trivia on the same physical line as the deliberately lazy import.
|
||||
- Fails closed on missing/unreadable inputs, TypeScript parse diagnostics, and scanner/process failures; comments, strings, templates, regex literals, and type-position `import(...)` syntax are not runtime imports.
|
||||
|
||||
- [ ] **Step 1: Record call-graph blast radius before touching functions**
|
||||
|
||||
First call `sources_list` and select the source whose registered path is this gbrain checkout. Then run `code_blast` and `code_callers` for these qualified symbols with that exact `source_id`, following `did_you_mean`/`candidates` when a method name is ambiguous:
|
||||
|
||||
```text
|
||||
src/core/pglite-engine.ts::PGLiteEngine.initSchema
|
||||
src/core/pglite-engine.ts::PGLiteEngine.batchRetry
|
||||
src/core/pglite-engine.ts::PGLiteEngine._upsertChunksOnce
|
||||
src/core/pglite-engine.ts::PGLiteEngine.mergeOntologyFact
|
||||
src/core/pglite-engine.ts::PGLiteEngine.getRecentSalience
|
||||
src/core/postgres-engine.ts::PostgresEngine.disconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.initSchema
|
||||
src/core/postgres-engine.ts::PostgresEngine.batchRetry
|
||||
src/core/postgres-engine.ts::PostgresEngine._upsertChunksOnce
|
||||
src/core/postgres-engine.ts::PostgresEngine.mergeOntologyFact
|
||||
src/core/postgres-engine.ts::PostgresEngine.reconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.getRecentSalience
|
||||
src/core/migrate.ts::runMigrationSQLWithRetry
|
||||
src/core/migrate.ts::runMigrations
|
||||
```
|
||||
|
||||
Use `depth: 5`, `max_nodes: 200`, and `limit: 100`. Expected: no caller requires a signature or behavior change; the patch only changes module binding time and retains all local fallback/error handling.
|
||||
|
||||
- [ ] **Step 2: Write the failing guard regression test**
|
||||
|
||||
Create `test/scripts/check-engine-dynamic-import.test.ts` as a hermetic subprocess suite. The completed 22-test surface covers:
|
||||
|
||||
- unmarked runtime `import()` rejection, including bare and trivia-separated forms;
|
||||
- same-line markers in real line or multiline block-comment trivia;
|
||||
- rejection of markers on prior lines or inside strings, templates, and module paths;
|
||||
- comments and comment-like delimiters inside strings, templates, and regex literals;
|
||||
- live code after same-line or multiline block comments close;
|
||||
- CRLF input and complete multi-file violation aggregation;
|
||||
- missing/readable mixed inputs and TypeScript parse diagnostics;
|
||||
- default repository anchoring when invoked from a foreign Git repository;
|
||||
- the reconciled three-file source scan plus package/parallel-verifier wiring.
|
||||
|
||||
Use the TypeScript parser rather than a partial lexical reimplementation. On Windows, set the test default to 30 seconds because each case launches Git Bash and Bun, whose startup can exceed Bun's 5-second per-test default.
|
||||
|
||||
- [ ] **Step 3: Run the test to prove the pre-implementation red state**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result captured inside the log. At minimum, the `exists` assertion fails because `scripts/check-engine-dynamic-import.sh` does not exist. Read `.context/engine-dynamic-import-red.txt`; do not infer the result from a truncated pipeline.
|
||||
|
||||
- [ ] **Step 4: Add the CRLF-safe, fail-closed guard**
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.sh` as a thin LF-terminated wrapper. Resolve its own directory first; when no explicit files are passed, anchor the repository with `git -C "$SCRIPT_DIR/.."` and scan the two engines plus `migrate.ts`. Delegate with `exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"` so scanner failures propagate.
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.ts` using the TypeScript compiler API:
|
||||
|
||||
- read every requested file and aggregate read failures;
|
||||
- parse as TypeScript and aggregate parse diagnostics;
|
||||
- walk the AST for `CallExpression`s whose expression is `ImportKeyword`;
|
||||
- locate all marker occurrences in the full source and use `ts.getTokenAtPosition` to admit only occurrences outside AST tokens (real comment trivia), recording their physical source lines;
|
||||
- require each runtime import's line to have an admitted marker or report its original `file:line:text`;
|
||||
- print every read/parse error and every violation before exiting nonzero.
|
||||
|
||||
This preserves CRLF line accounting, ignores comment/literal/type-only false positives, catches every legal runtime `import()` shape the TypeScript parser recognizes, rejects marker spoofing, and fails closed.
|
||||
|
||||
- [ ] **Step 5: Run the guard test to prove the source-tree midpoint is still red**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-midpoint.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: the synthetic violation, marker, comments, and CRLF cases pass. The default repository scan fails and reports all 17 current imports: 13 unmarked safe candidates plus the four not-yet-marked gateway calls.
|
||||
|
||||
- [ ] **Step 6: Hoist the three safe PGLite import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add the ontology/recency imports near the top of `src/core/pglite-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
```
|
||||
|
||||
Delete only these three in-method destructuring imports, leaving their uses unchanged:
|
||||
|
||||
```ts
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Mark both PGLite gateway soft-failure boundaries**
|
||||
|
||||
In `PGLiteEngine.initSchema`, preserve the `try/catch` and accessors, changing only the rationale and import line:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PGLiteEngine._upsertChunksOnce`, preserve the config-row and compile-time fallback chain:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Hoist the eight safe Postgres import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add these imports near the top of `src/core/postgres-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import { isConnectionEndedError } from './retry-matcher.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { logDbDisconnect } from './audit/db-disconnect-audit.ts';
|
||||
import { logPoolRecovery } from './audit/pool-recovery-audit.ts';
|
||||
```
|
||||
|
||||
Delete the eight safe dynamic-import statements while keeping their surrounding `try/catch` blocks and calls unchanged:
|
||||
|
||||
```ts
|
||||
const { logDbDisconnect } = await import('./audit/db-disconnect-audit.ts');
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { isConnectionEndedError } = await import('./retry-matcher.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
Update the stale `batchRetry` comment from “Lazy-import to avoid a circular dep concern” to current truth:
|
||||
|
||||
```ts
|
||||
// retry.ts is already in this module's static graph through withRetry, so
|
||||
// classifying the exhausted error does not need a second runtime import.
|
||||
```
|
||||
|
||||
- [ ] **Step 9: Mark both Postgres gateway soft-failure boundaries**
|
||||
|
||||
In `PostgresEngine.initSchema`, mirror the PGLite rationale and preserve behavior:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PostgresEngine._upsertChunksOnce`, preserve the DB-config fallback:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 10: Hoist the two migration helper import statements**
|
||||
|
||||
Add these static imports at the top of `src/core/migrate.ts`:
|
||||
|
||||
```ts
|
||||
// runMigrations executes while an initialized engine is live. Keep its helper
|
||||
// modules in the static graph rather than importing them from async handlers.
|
||||
import {
|
||||
isStatementTimeoutError,
|
||||
isRetryableConnError,
|
||||
} from './retry-matcher.ts';
|
||||
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
|
||||
```
|
||||
|
||||
Delete only these two local destructuring imports:
|
||||
|
||||
```ts
|
||||
const { isStatementTimeoutError, isRetryableConnError } = await import('./retry-matcher.ts');
|
||||
const { repairTimelineDedupIndex } = await import('./timeline-dedup-repair.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 11: Run the complete guard test and direct guard**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; output contains `check-engine-dynamic-import: ok (3 file(s) scanned)`.
|
||||
|
||||
- [ ] **Step 12: Prove the guard leaves exactly four marked dynamic imports**
|
||||
|
||||
```bash
|
||||
git grep -n -F "import('./ai/gateway.ts'); // engine-dynamic-import-ok" -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts > .context/engine-dynamic-import-sites.txt; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exactly four lines, all importing `./ai/gateway.ts` and all carrying `engine-dynamic-import-ok`; no match in `src/core/migrate.ts`.
|
||||
|
||||
- [ ] **Step 13: Run focused behavior tests**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. If Windows resource pressure aborts the process, record the exact exit code and rerun the failing file alone; do not relabel an infrastructure abort as a source pass.
|
||||
|
||||
- [ ] **Step 14: Commit the source invariant locally**
|
||||
|
||||
```bash
|
||||
git add scripts/check-engine-dynamic-import.sh scripts/check-engine-dynamic-import.ts test/scripts/check-engine-dynamic-import.test.ts src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "fix(engine): reconcile dynamic import hardening"
|
||||
```
|
||||
|
||||
Expected: one local commit; no version or release files staged.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Wire the guard into repository checks
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `package.json`
|
||||
- Modify: `scripts/run-verify-parallel.sh`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `scripts/check-engine-dynamic-import.sh` from Task 1.
|
||||
- Produces: package script `check:engine-dynamic-import` and verify dry-list entry of the same name.
|
||||
|
||||
- [ ] **Step 1: Add failing wiring assertions**
|
||||
|
||||
Add these imports/constants to `test/scripts/check-engine-dynamic-import.test.ts`:
|
||||
|
||||
```ts
|
||||
const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json');
|
||||
```
|
||||
|
||||
Append this test block:
|
||||
|
||||
```ts
|
||||
describe('engine dynamic-import guard wiring', () => {
|
||||
it('is invoked through bash by check:all', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf8')) as {
|
||||
scripts: Record<string, string>;
|
||||
};
|
||||
expect(pkg.scripts['check:engine-dynamic-import']).toBe(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
expect(pkg.scripts['check:all']).toContain(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
});
|
||||
|
||||
it('is listed by the authoritative verify dispatcher', () => {
|
||||
const result = spawnSync(BASH, [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(new Set((result.stdout ?? '').trim().split('\n'))).toContain(
|
||||
'check:engine-dynamic-import',
|
||||
);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the test and verify both wiring assertions fail**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result. The source guard tests remain green; package-script and verify-list assertions fail because the wiring is absent.
|
||||
|
||||
- [ ] **Step 3: Add the package scripts**
|
||||
|
||||
In `package.json`, add this script alongside the other `check:*` entries:
|
||||
|
||||
```json
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh"
|
||||
```
|
||||
|
||||
Append the guard to the existing `check:all` chain, preserving every existing check:
|
||||
|
||||
```text
|
||||
&& bash scripts/check-engine-dynamic-import.sh
|
||||
```
|
||||
|
||||
Do not rewrite any existing shell entry without its `bash` prefix.
|
||||
|
||||
- [ ] **Step 4: Add the authoritative verify entry**
|
||||
|
||||
In `scripts/run-verify-parallel.sh`, add this stable `CHECKS` entry near the other source-shape guards:
|
||||
|
||||
```bash
|
||||
"check:engine-dynamic-import"
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run the regression test and package check**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bun run check:engine-dynamic-import > .context/engine-dynamic-import-package-check.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0` and three files scanned.
|
||||
|
||||
- [ ] **Step 6: Commit the wiring locally**
|
||||
|
||||
```bash
|
||||
git add package.json scripts/run-verify-parallel.sh test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "test(engine): guard dynamic import policy"
|
||||
```
|
||||
|
||||
Expected: one local commit with the guard wiring and its regression assertions.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Document the current-state invariant
|
||||
|
||||
**Files:**
|
||||
- Modify: `CLAUDE.md`
|
||||
- Modify: `docs/architecture/KEY_FILES.md`
|
||||
- Regenerate: `llms.txt`
|
||||
- Regenerate: `llms-full.txt`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the four-marked-import source state and the `check:engine-dynamic-import` package surface.
|
||||
- Produces: current-state contributor guidance and fresh generated documentation bundles.
|
||||
|
||||
- [ ] **Step 1: Add the cross-cutting invariant to `CLAUDE.md`**
|
||||
|
||||
Add this bullet under “Cross-cutting invariants” near the other language/filesystem guards:
|
||||
|
||||
```md
|
||||
- **Engine-live paths use static imports by default.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, helper modules are top-level imports. The only current
|
||||
exceptions are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
```
|
||||
|
||||
Do not add release tags, Windows-crash certainty, or historical branch names.
|
||||
|
||||
- [ ] **Step 2: Update the PGLite current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this current-state sentence to the existing `src/core/pglite-engine.ts` entry, preserving the entry as one bullet:
|
||||
|
||||
```md
|
||||
Engine-path helper dependencies (`retry`, ontology, recency decay) bind statically; the only lazy imports are `ai/gateway.ts` in `initSchema` and `_upsertChunksOnce`, line-marked because their local catches preserve compiled-default and stored-config fallbacks that eager module evaluation would bypass.
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update the Postgres current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the existing `src/core/postgres-engine.ts` entry:
|
||||
|
||||
```md
|
||||
Retry classifiers, ontology/recency helpers, and disconnect/pool-recovery audit writers bind statically; only the two `ai/gateway.ts` fallback lookups stay lazy and line-marked, in parity with PGLite.
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update the migration current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the canonical `src/core/migrate.ts` entry (the broad runner entry, not the older v95-specific index note):
|
||||
|
||||
```md
|
||||
`retry-matcher.ts` and `timeline-dedup-repair.ts` are static dependencies because `runMigrations()` executes from live engine initialization; the engine dynamic-import guard scans this file with both engine implementations.
|
||||
```
|
||||
|
||||
Keep all three entries current-state only: no `v0.42.x`, branch, commit, “previously,” or “was/now” narration.
|
||||
|
||||
- [ ] **Step 5: Regenerate the llms bundles**
|
||||
|
||||
```bash
|
||||
bun run build:llms > .context/engine-dynamic-import-build-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; `llms.txt` and/or `llms-full.txt` update according to their configured linked/inlined status. Byte-identical output for a linked source is acceptable; the freshness test is authoritative.
|
||||
|
||||
- [ ] **Step 6: Run documentation freshness checks**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-llms-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
```bash
|
||||
bun run check:doc-history > .context/engine-dynamic-import-doc-history.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; no release-history marker is introduced into current-state reference docs.
|
||||
|
||||
- [ ] **Step 7: Confirm prohibited release files remain untouched**
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD -- VERSION CHANGELOG.md TODOS.md
|
||||
```
|
||||
|
||||
Expected: no output.
|
||||
|
||||
- [ ] **Step 8: Commit documentation and generated bundles locally**
|
||||
|
||||
```bash
|
||||
git add CLAUDE.md docs/architecture/KEY_FILES.md llms.txt llms-full.txt
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs(engine): record static import invariant"
|
||||
```
|
||||
|
||||
Expected: one local documentation commit. If one generated bundle is byte-identical, Git simply omits it.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Verify and review the complete local reconciliation
|
||||
|
||||
**Files:**
|
||||
- Verify all files changed since `d7f52d8c`.
|
||||
- Do not create or modify release/publication metadata.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Tasks 1–3.
|
||||
- Produces: full local verification evidence and an implementation diff ready for user review, not publication.
|
||||
|
||||
- [ ] **Step 1: Run the regression test and direct guard again**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-final-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-final-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; three files scanned.
|
||||
|
||||
- [ ] **Step 2: Run TypeScript checking**
|
||||
|
||||
```bash
|
||||
bun run typecheck > .context/engine-dynamic-import-typecheck.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. Report exact diagnostics if the branch or current Windows environment has a pre-existing failure.
|
||||
|
||||
- [ ] **Step 3: Run the authoritative verify dispatcher**
|
||||
|
||||
```bash
|
||||
bun run verify > .context/engine-dynamic-import-verify.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`, including `check:engine-dynamic-import`. On Windows, classify any per-check timeout from the complete log instead of treating the aggregate result as a source regression without evidence.
|
||||
|
||||
- [ ] **Step 4: Re-run focused tests as an ownership check**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-final-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; record any infrastructure abort separately and rerun only the named file before classifying it.
|
||||
|
||||
- [ ] **Step 5: Run the llms freshness test after all documentation settles**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-final-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
- [ ] **Step 6: Run whitespace and scope checks**
|
||||
|
||||
```bash
|
||||
git diff --check d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected: exit `0`, no output.
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected files only:
|
||||
|
||||
```text
|
||||
CLAUDE.md
|
||||
docs/architecture/KEY_FILES.md
|
||||
docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
llms-full.txt
|
||||
llms.txt
|
||||
package.json
|
||||
scripts/check-engine-dynamic-import.sh
|
||||
scripts/check-engine-dynamic-import.ts
|
||||
scripts/run-verify-parallel.sh
|
||||
src/core/migrate.ts
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
Either generated llms file may be absent if regeneration proves it byte-identical. `VERSION`, `CHANGELOG.md`, and `TODOS.md` must be absent.
|
||||
|
||||
- [ ] **Step 7: Review the exact implementation diff**
|
||||
|
||||
```bash
|
||||
git diff --stat d7f52d8c..HEAD && git diff d7f52d8c..HEAD -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts scripts/check-engine-dynamic-import.sh test/scripts/check-engine-dynamic-import.test.ts package.json scripts/run-verify-parallel.sh CLAUDE.md docs/architecture/KEY_FILES.md
|
||||
```
|
||||
|
||||
Expected review findings:
|
||||
|
||||
- Exactly 13 safe `await import(...)` statements are removed.
|
||||
- Exactly four `ai/gateway.ts` imports remain, all marked on the same line.
|
||||
- All four gateway imports remain inside their original local `try/catch` fallback boundaries.
|
||||
- No accessor logic, fallback ordering, SQL, public signature, or engine parity behavior changes.
|
||||
- The parser-backed guard reports all violations plus read/parse failures, preserves CRLF line accounting, ignores comments/literals/type-only syntax, detects every runtime `import()` call expression, and accepts opt-outs only from real comment trivia on the same physical line.
|
||||
- The package script invokes the shell guard through Bash; `check:all` invokes that shell guard directly, and the parallel verify dispatcher invokes the package check.
|
||||
- Documentation is current-state and makes no deterministic Windows-crash claim.
|
||||
|
||||
**Observed Windows verification classification:** The authoritative aggregate completed with 25 of 33 checks passing. Individual reruns showed `check:test-names` and `typecheck` green; privacy/isolation exceeded Windows timing budgets; WASM failed in unrelated temporary-symlink setup; eval-glossary was CRLF/LF drift; resolver/brain-first findings predated and did not intersect this branch. The focused aggregate produced 103 pass / 5 fail: three setup-hook timeouts reproduced at the untouched base, and the known `migrate-retry` polling failure reproduced there. Its additional race-status assertion did not reproduce at base, so it remains an unresolved timing-sensitive limitation in untouched code—not evidence of an in-scope defect and not claimed as conclusively pre-existing.
|
||||
|
||||
- [ ] **Step 8: Commit the approved plan document locally**
|
||||
|
||||
The plan is an approved, tracked execution artifact and must not be left as an uncommitted file after implementation:
|
||||
|
||||
```bash
|
||||
git add docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs: plan engine dynamic-import reconciliation"
|
||||
```
|
||||
|
||||
Expected: one local plan commit; no release metadata staged.
|
||||
|
||||
- [ ] **Step 9: Inspect final status without publishing**
|
||||
|
||||
```bash
|
||||
git status --short --branch
|
||||
```
|
||||
|
||||
Expected: branch `claude/kind-meitner-330c90` with a clean working tree. No push, PR, upstream comment, or other external side effect.
|
||||
|
||||
- [ ] **Step 10: Capture the completed milestone to memory**
|
||||
|
||||
Before writing, search MemPalace wing `gbrain` for this exact reconciliation to avoid duplication. Add a verbatim drawer recording exact base/head commits, the 13 hoists, four gateway opt-outs and rationale, guard/test/docs files, every verification command with exit code, and any environment-owned failures. Add a GBrain project timeline entry only if there is an existing relevant gbrain project page; do not create duplicate release metadata.
|
||||
|
||||
- [ ] **Step 11: Report the local result and ask separately before publication**
|
||||
|
||||
Report:
|
||||
|
||||
- exact local commits;
|
||||
- changed files;
|
||||
- test/check exit codes;
|
||||
- any blocked or pre-existing failures;
|
||||
- confirmation that release files were untouched;
|
||||
- confirmation that nothing was pushed or published.
|
||||
|
||||
Do not run any publication command. Wait for explicit user approval before any push, PR, or upstream interaction.
|
||||
@@ -1,175 +0,0 @@
|
||||
# Scalar-source Backlink Validation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make backlink validation compare exact `(source_id, slug)` endpoint identities while preserving existing scalar, unscoped, and federated link-read semantics.
|
||||
|
||||
**Architecture:** Enrich every engine link-read row with the source identity of its joined from, to, and visible origin pages. Pass the validated page's scalar or federated scope into validator context; the backlink validator scopes its initial read consistently, groups targets by exact identity, and accepts only an exact reverse endpoint pair. SQL predicates remain unchanged, so trusted scalar cross-source visibility and federated all-endpoint containment remain intact.
|
||||
|
||||
**Tech Stack:** TypeScript, Bun test, PGLite, PostgreSQL/postgres.js.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Use strict red-before-green TDD with duplicate slugs across sources.
|
||||
- Preserve unscoped historical reads, scalar near-endpoint scoping, scalar explicit cross-source visibility, federated all-endpoint containment, and `sourceIds` precedence.
|
||||
- Keep PostgreSQL and PGLite projections in parity.
|
||||
- Do not change schema or conditional-write conflict semantics.
|
||||
- Keep deployment, restart, migration, and push actions outside the implementation tasks; a separately authorized release workflow may perform them after verification.
|
||||
- Capture full test output to files before inspecting it.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Pin the backlink false-negative in PGLite
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/writer.test.ts`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backLinkValidator.validate(PageValidationContext)` and source-qualified `putPage`/`addLink`.
|
||||
- Produces: regressions for wrong-source reverse rejection, exact reverse acceptance, cross-source pair acceptance, and exact target deduplication.
|
||||
|
||||
- [ ] **Step 1: Add the minimal failing duplicate-slug regression**
|
||||
|
||||
Create `default` and `team-x` copies of the origin and target, add `(team-x, origin) -> (team-x, target)` plus the wrong reverse `(team-x, target) -> (default, origin)`, validate with `sourceId: 'team-x'`, and require one warning.
|
||||
|
||||
- [ ] **Step 2: Run the focused test and verify RED**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts -t "wrong-source reverse" > "$TEMP/backlink-red.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: assertion failure because current slug-only validation returns zero findings.
|
||||
|
||||
- [ ] **Step 3: Add the remaining behavioral regressions after the first red is recorded**
|
||||
|
||||
Add tests proving that the exact reverse clears the warning, a legitimate cross-source forward/reverse pair passes, and two destinations sharing one slug but differing by source are validated independently.
|
||||
|
||||
### Task 2: Expose exact endpoint identity from both engines
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/core/types.ts:1204-1229`
|
||||
- Modify: `src/core/postgres-engine.ts:3021-3124`
|
||||
- Modify: `src/core/pglite-engine.ts:2941-3037`
|
||||
- Modify: `test/get-page-federated-scope.test.ts:187-246,289-306`
|
||||
- Modify: `test/e2e/multi-source-bug-class.test.ts:184-205`
|
||||
- Modify: `test/e2e/engine-parity.test.ts:813-875`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `Link.from_source_id: string`, `Link.to_source_id: string`, and `Link.origin_source_id?: string | null`.
|
||||
- Preserves: `getLinks(slug, { sourceId?, sourceIds? })` and `getBacklinks(...)` filtering semantics.
|
||||
|
||||
- [ ] **Step 1: Add engine-contract assertions before implementation**
|
||||
|
||||
Assert scalar cross-source rows expose `beta -> default`, federated rows expose only in-grant endpoint IDs, `sourceIds` still beats scalar `sourceId`, and an out-of-grant origin has both `origin_slug` and `origin_source_id` null.
|
||||
|
||||
- [ ] **Step 2: Run the focused contract tests and verify RED**
|
||||
|
||||
```bash
|
||||
bun test test/get-page-federated-scope.test.ts test/e2e/multi-source-bug-class.test.ts > "$TEMP/link-identity-red.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: source-ID assertions fail because fields are absent.
|
||||
|
||||
- [ ] **Step 3: Extend `Link` and project IDs without changing predicates**
|
||||
|
||||
Use this additive contract:
|
||||
|
||||
```ts
|
||||
export interface Link {
|
||||
from_slug: string;
|
||||
from_source_id: string;
|
||||
to_slug: string;
|
||||
to_source_id: string;
|
||||
link_type: string;
|
||||
context: string;
|
||||
link_source?: string | null;
|
||||
origin_slug?: string | null;
|
||||
origin_source_id?: string | null;
|
||||
origin_field?: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
In all six branches per engine, project:
|
||||
|
||||
```sql
|
||||
f.source_id AS from_source_id,
|
||||
t.source_id AS to_source_id,
|
||||
o.source_id AS origin_source_id
|
||||
```
|
||||
|
||||
Keep every `WHERE` and grant-aware origin `LEFT JOIN` unchanged.
|
||||
|
||||
- [ ] **Step 4: Re-run contract tests and verify GREEN**
|
||||
|
||||
Use the same command and require all focused tests to pass.
|
||||
|
||||
### Task 3: Validate exact reverse identities and propagate scope
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/core/output/writer.ts:89-96,240-318`
|
||||
- Modify: `src/core/output/post-write.ts:36-41,73-118`
|
||||
- Modify: `src/core/output/validators/back-link.ts:24-47`
|
||||
- Modify: `src/core/operations.ts:1227-1246`
|
||||
- Modify: `test/post-write-lint.test.ts:67-130`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: optional `PageValidationContext.sourceId` and `sourceIds`, with `sourceIds` taking precedence.
|
||||
- `runPostWriteLint(..., opts)` accepts the same optional scope and loads the validated page through it.
|
||||
|
||||
- [ ] **Step 1: Add a post-write nested-read regression and verify RED**
|
||||
|
||||
Validate a non-default page with a wrong-source reverse via `runPostWriteLint(..., { force: true, noLog: true, sourceId: 'team-x' })`; require a backlink warning.
|
||||
|
||||
- [ ] **Step 2: Implement minimal scope propagation**
|
||||
|
||||
Add `sourceId?`/`sourceIds?` to validation context and lint options. Load pages using `sourceIds` when non-empty, otherwise scalar `sourceId`. Pass the same scope into nested validators. In the put-page success hook, call lint with the already-resolved write source ID.
|
||||
|
||||
- [ ] **Step 3: Implement exact backlink matching**
|
||||
|
||||
Initial outbound reads use the validation scope. Deduplicate rows by all four endpoint identity fields so every distinct expected origin remains represented even when targets share a source-qualified identity. Read each target using the federated grant when present, otherwise the target's exact scalar source. Accept only a row matching all four endpoint fields of the expected reverse.
|
||||
|
||||
- [ ] **Step 4: Run writer and post-write tests and verify GREEN**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts test/post-write-lint.test.ts > "$TEMP/backlink-green.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: all tests pass, including the recorded false-negative.
|
||||
|
||||
### Task 4: Verify PostgreSQL/PGLite parity and final scope
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/e2e/engine-parity.test.ts:813-875`
|
||||
- Verify: all files above
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: exact endpoint fields and unchanged filtering semantics.
|
||||
- Produces: parity evidence for scalar cross-source and federated reads.
|
||||
|
||||
- [ ] **Step 1: Compare complete endpoint tuples across engines**
|
||||
|
||||
Compare sorted tuples containing `from_source_id`, `from_slug`, `to_source_id`, `to_slug`, `origin_source_id`, and `origin_slug` for scalar and federated fixtures.
|
||||
|
||||
- [ ] **Step 2: Run focused PGLite/source-isolation tests**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts test/post-write-lint.test.ts test/get-page-federated-scope.test.ts test/e2e/multi-source-bug-class.test.ts > "$TEMP/backlink-focused.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0.
|
||||
|
||||
- [ ] **Step 3: Run PostgreSQL parity when the test database is available**
|
||||
|
||||
```bash
|
||||
bun test test/e2e/engine-parity.test.ts -t "federated sourceIds" --timeout=300000 > "$TEMP/backlink-parity.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0; if the configured test database is unavailable, report the exact environmental blocker rather than claiming parity execution.
|
||||
|
||||
- [ ] **Step 4: Typecheck and inspect the final diff**
|
||||
|
||||
```bash
|
||||
bun run typecheck > "$TEMP/backlink-typecheck.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0. Then run `git diff --check` and confirm no version, schema, migration, deployment, or conditional-write files changed.
|
||||
@@ -1,142 +0,0 @@
|
||||
# Engine dynamic-import reconciliation design
|
||||
|
||||
**Date:** 2026-07-28
|
||||
|
||||
## Goal
|
||||
|
||||
Reconcile the overlapping engine dynamic-import changes from:
|
||||
|
||||
- `claude/hungry-edison-8bb1cd` at release commits `48ada48f` and `248bfe55`
|
||||
- `claude/elegant-gates-e5275e` at `ef4cf7a8`
|
||||
|
||||
onto a fresh branch from current `origin/master`, without merging or cherry-picking either lineage wholesale and without adding a release/version bump.
|
||||
|
||||
## Established state
|
||||
|
||||
At investigation time:
|
||||
|
||||
- `origin/master` was `6136e139972a5449630b4f47f5ed7b4cbe5b811b`, version `0.42.67.0`.
|
||||
- Upstream PR #3511 was still open, so trunk did not contain its two `chronicle/ontology.ts` hoists.
|
||||
- Neither source branch was an ancestor of trunk.
|
||||
- Trunk contained 17 dynamic imports in the three engine-path files:
|
||||
- 13 safe-hoist candidates: two ontology imports, nine engine helper/audit imports, and two migration imports.
|
||||
- Four `ai/gateway.ts` imports, all inside `try/catch` fallback paths.
|
||||
- `git log -G` showed the separate ontology, helper, migration, and gateway histories. `git log -S` is not suitable for this dynamic-to-static replacement because the relevant token can remain present while its context changes.
|
||||
- The guard from `ef4cf7a8` passed against that commit but failed against trunk. It also knew about only two gateway opt-outs because two `_upsertChunksOnce` gateway lookups landed later in trunk.
|
||||
|
||||
## Selected approach
|
||||
|
||||
Reconstruct the intended current state directly on fresh `origin/master`.
|
||||
|
||||
Do not merge or cherry-pick either old lineage. Selectively reproduce the desired source changes, adapt the guard to the current four gateway call sites, and write current-state documentation. This avoids importing stale release metadata, stale TODO claims, and unrelated lineage changes.
|
||||
|
||||
## Source changes
|
||||
|
||||
### Safe static imports
|
||||
|
||||
Hoist all 13 safe candidates:
|
||||
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `valueHash`, `normalizeDimension`, `isNovelDimension` from `chronicle/ontology.ts`
|
||||
- `isRetryableConnError` through the existing `retry.ts` import
|
||||
- `resolveRecencyDecayMap`, `DEFAULT_FALLBACK` from `search/recency-decay.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- the same ontology, retry, and recency helpers
|
||||
- `isConnectionEndedError` from `retry-matcher.ts`
|
||||
- `logDbDisconnect` from `audit/db-disconnect-audit.ts`
|
||||
- `logPoolRecovery` from `audit/pool-recovery-audit.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- `isStatementTimeoutError`, `isRetryableConnError` from `retry-matcher.ts`
|
||||
- `repairTimelineDedupIndex` from `timeline-dedup-repair.ts`
|
||||
|
||||
The implementation must keep the two engines in parity where the behavior is shared. Comments should describe current invariants, not repeat an unproven causal claim that these hoists fix the Windows test-runner crash.
|
||||
|
||||
### Deliberately lazy gateway imports
|
||||
|
||||
Keep all four `await import('./ai/gateway.ts')` call sites lazy:
|
||||
|
||||
- PGLite `initSchema`
|
||||
- PGLite `_upsertChunksOnce`
|
||||
- Postgres `initSchema`
|
||||
- Postgres `_upsertChunksOnce`
|
||||
|
||||
Each line receives the explicit `engine-dynamic-import-ok` marker and a concise nearby rationale.
|
||||
|
||||
The rationale has two parts:
|
||||
|
||||
1. The gateway's static closure includes the AI SDK, provider packages, and validation/config machinery, so eager loading would tax engine startup paths that do not otherwise need it.
|
||||
2. More importantly, each lookup is inside a `try/catch` that preserves a soft fallback (compiled defaults or the brain's stored embedding-model config). Hoisting the module would evaluate it before that catch can run and could convert a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
|
||||
The guard must not allow unmarked gateway imports or a broad file-level exemption.
|
||||
|
||||
## Guard and wiring
|
||||
|
||||
Add `scripts/check-engine-dynamic-import.sh`, adapted from `ef4cf7a8`, with these properties:
|
||||
|
||||
- Default scan set:
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- Normalize trailing CR before matching so CRLF checkouts cannot bypass the check.
|
||||
- Ignore comment-only lines.
|
||||
- Ignore only lines carrying `engine-dynamic-import-ok`.
|
||||
- Report every unmarked `await import(` with file and line.
|
||||
- Explain that contributors should prefer a static import and must justify a real opt-out.
|
||||
- Avoid asserting that every dynamic import deterministically crashes Windows; the measured evidence supports treating the pattern as an engine-path hardening invariant, while box-level commit exhaustion remained a confound in prior runs.
|
||||
|
||||
Wire it into:
|
||||
|
||||
- `package.json` as `check:engine-dynamic-import`
|
||||
- `package.json` `check:all`
|
||||
- `scripts/run-verify-parallel.sh`
|
||||
|
||||
Follow trunk's current rule that package scripts invoke repository shell scripts through `bash`.
|
||||
|
||||
## Regression coverage
|
||||
|
||||
Add an automated test for the guard. It must cover:
|
||||
|
||||
- A real dynamic import produces exit 1 and is reported.
|
||||
- A line carrying `engine-dynamic-import-ok` is allowed.
|
||||
- Line comments and block-comment lines do not produce findings.
|
||||
- The same violation is caught with CRLF input.
|
||||
- The default repository scan passes after the source reconciliation.
|
||||
|
||||
Use a temporary fixture rather than mutating tracked source files. Keep assertions path-portable.
|
||||
|
||||
The pre-fix red demonstration is the exact guard from `ef4cf7a8` run against current trunk: it exits 1 and reports the existing unmarked imports. The post-fix guard and test must pass.
|
||||
|
||||
## Documentation policy
|
||||
|
||||
Preserve current behavior, not either old release narrative:
|
||||
|
||||
- Do not modify `VERSION` or add a release `CHANGELOG.md` entry.
|
||||
- Do not copy old version headings or completed release TODO blocks.
|
||||
- Do not retain the old TODO claiming that extracting gateway accessors is necessarily the fix; the lazy imports are deliberately protected by their local soft-failure boundaries.
|
||||
- Add the cross-cutting no-unmarked-dynamic-import invariant to `CLAUDE.md`.
|
||||
- Update the current-state entries for `src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and `src/core/migrate.ts` in `docs/architecture/KEY_FILES.md` where needed.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` after the documentation edits.
|
||||
- Add a TODO only if implementation uncovers a real unresolved action.
|
||||
|
||||
Public documentation must use generic language and must not overstate the historical Windows crash causality.
|
||||
|
||||
## Verification
|
||||
|
||||
Capture full output to files before inspecting summaries. Run, at minimum:
|
||||
|
||||
1. The guard regression test.
|
||||
2. `bash scripts/check-engine-dynamic-import.sh`.
|
||||
3. Focused tests that exercise the touched engine, migration, retry, audit, and recency modules.
|
||||
4. `bun run typecheck`.
|
||||
5. `bun run verify`.
|
||||
6. `bun run build:llms` followed by `bun test test/build-llms.test.ts`.
|
||||
7. `git diff --check` and a final clean-status/diff review.
|
||||
|
||||
If platform contention or existing Windows suite defects block a broad test, report the exact command, exit code, and ownership classification rather than declaring success from a partial run.
|
||||
|
||||
## Git and publication boundary
|
||||
|
||||
- Work on `claude/kind-meitner-330c90`, reset locally to the exact investigated `origin/master` base.
|
||||
- Preserve the previous worktree tip under `claude/kind-meitner-330c90-pre-reconcile`.
|
||||
- Keep implementation and verification commits local.
|
||||
- Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after the local result is complete.
|
||||
@@ -1,184 +0,0 @@
|
||||
# Scalar-source backlink validation design
|
||||
|
||||
## Problem
|
||||
|
||||
A page identity in a multi-source brain is `(source_id, slug)`, but the back-link validator currently reasons only about `slug`.
|
||||
|
||||
For an outbound edge:
|
||||
|
||||
```text
|
||||
(source-a, concepts/origin) -> (source-a, people/target)
|
||||
```
|
||||
|
||||
the validator accepts any reverse row whose bare slugs are:
|
||||
|
||||
```text
|
||||
people/target -> concepts/origin
|
||||
```
|
||||
|
||||
That can incorrectly accept a row ending at `(default, concepts/origin)` instead of `(source-a, concepts/origin)`.
|
||||
|
||||
The bug is not that scalar `getLinks(slug, { sourceId })` permits cross-source destinations. That behavior is intentional: scalar scope qualifies the near/from endpoint while trusted local callers retain visibility into explicit cross-source edges. The gap is that a returned `Link` does not carry the source identity of either endpoint, so callers cannot distinguish same-slug pages.
|
||||
|
||||
## Reproduction and evidence
|
||||
|
||||
A deterministic PGLite reproduction creates duplicate `concepts/a` and `people/b` pages in `default` and `team-x`, then adds:
|
||||
|
||||
```text
|
||||
(team-x, concepts/a) -> (team-x, people/b)
|
||||
(team-x, people/b) -> (default, concepts/a)
|
||||
```
|
||||
|
||||
The second edge is not a valid reverse of the first. Nevertheless:
|
||||
|
||||
```ts
|
||||
await engine.getLinks('people/b', { sourceId: 'team-x' })
|
||||
```
|
||||
|
||||
returns the second row, and the current validator accepts it because `to_slug === 'concepts/a'`.
|
||||
|
||||
Both engines implement the same scalar rule: filter `f.slug` and `f.source_id`, join the actual destination by `to_page_id`, and do not filter `t.source_id`. Federated `sourceIds` is a separate branch that constrains all visible endpoints and takes precedence over scalar scope.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Validate back-links by exact source-qualified endpoint identity.
|
||||
2. Preserve explicit cross-source links for trusted scalar reads.
|
||||
3. Preserve federated all-endpoint containment and `sourceIds` precedence.
|
||||
4. Keep PostgreSQL and PGLite behavior identical.
|
||||
5. Add strict red-before-green regressions using duplicate slugs across sources.
|
||||
6. Avoid schema migrations and production operational changes.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing scalar link reads to same-source-only reads.
|
||||
- Weakening or widening federated reads.
|
||||
- Changing link write identity or database schema.
|
||||
- Refactoring the atomic conditional-write branch.
|
||||
- Coupling deployment, restart, or migration mechanics to the backlink code change. Release operations are handled separately after verification.
|
||||
|
||||
## Chosen approach
|
||||
|
||||
Extend the engine `Link` result with endpoint source identities and use those fields in the validator.
|
||||
|
||||
```ts
|
||||
interface Link {
|
||||
from_slug: string;
|
||||
from_source_id: string;
|
||||
to_slug: string;
|
||||
to_source_id: string;
|
||||
// existing fields
|
||||
origin_slug?: string | null;
|
||||
origin_source_id?: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
All `getLinks` and `getBacklinks` query branches in PostgreSQL and PGLite will project the source IDs from the pages already joined as `f`, `t`, and `o`. No filtering behavior changes.
|
||||
|
||||
This approach is preferred over a dedicated `hasExactLink` method because it keeps source identity attached to the link data everywhere, avoids duplicate engine SQL and per-edge existence queries, and matches existing source-qualified link-write and batch-row contracts.
|
||||
|
||||
Validator-only raw SQL is rejected because validators should consume the `BrainEngine` contract rather than bypass it with engine-specific schema knowledge.
|
||||
|
||||
## Engine semantics
|
||||
|
||||
The existing three read modes remain unchanged.
|
||||
|
||||
### Unscoped
|
||||
|
||||
`getLinks(slug)` returns rows from all same-slug from-pages across sources. Each row identifies the actual source of both endpoints.
|
||||
|
||||
### Scalar source
|
||||
|
||||
`getLinks(slug, { sourceId })` matches exactly `(sourceId, slug)` on the from side. A destination may belong to another source, and `to_source_id` reveals that exact identity.
|
||||
|
||||
The corresponding scalar `getBacklinks` rule continues to match the exact destination/to-page identity while allowing a cross-source referrer.
|
||||
|
||||
### Federated sources
|
||||
|
||||
`getLinks(slug, { sourceIds })` continues to constrain from and to endpoints to the grant. The origin join continues to redact an out-of-grant origin. `sourceIds` continues to take precedence over scalar `sourceId`.
|
||||
|
||||
Adding source IDs to returned in-grant endpoints does not disclose anything new: the existing result already discloses those pages' slugs and edges. An out-of-grant endpoint remains absent.
|
||||
|
||||
## Validator algorithm
|
||||
|
||||
The validator receives the source scope associated with the page being validated.
|
||||
|
||||
For every outbound edge:
|
||||
|
||||
```text
|
||||
(from_source_id, from_slug) -> (to_source_id, to_slug)
|
||||
```
|
||||
|
||||
it requires a reverse row:
|
||||
|
||||
```text
|
||||
(to_source_id, to_slug) -> (from_source_id, from_slug)
|
||||
```
|
||||
|
||||
Duplicate edge rows are deduplicated by the full endpoint pair `(from_source_id, from_slug, to_source_id, to_slug)`, not by bare target slug. This preserves separate reverse requirements when multiple same-slug origin pages point to one exact target.
|
||||
|
||||
For each target:
|
||||
|
||||
1. Read target outbound links using the target's exact scalar source when validation is scalar-scoped.
|
||||
2. Under federated validation, retain the caller's `sourceIds` grant rather than converting it to scalar scope.
|
||||
3. Accept only a returned row whose `from_source_id`, `from_slug`, `to_source_id`, and `to_slug` exactly match the expected reverse identity.
|
||||
4. Emit the existing warning when no exact reverse exists.
|
||||
|
||||
This preserves legitimate cross-source pairs. For example:
|
||||
|
||||
```text
|
||||
(source-a, concepts/origin) -> (source-b, people/target)
|
||||
(source-b, people/target) -> (source-a, concepts/origin)
|
||||
```
|
||||
|
||||
is valid.
|
||||
|
||||
## Validation context propagation
|
||||
|
||||
`PageValidationContext` must carry the relevant scalar or federated source scope. The writer and post-write lint paths must load the page with that scope and pass the same scope to nested validator reads.
|
||||
|
||||
This change is scoped to source routing needed by validation. It does not modify conditional-write revision or conflict semantics and must not be applied to the atomic conditional-write branch.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
### PGLite strict-TDD regression
|
||||
|
||||
Add duplicate pages across `default` and a second source, then prove before the production fix that:
|
||||
|
||||
1. A forward edge in the second source plus a wrong-source reverse produces a warning.
|
||||
2. Adding the exact reverse removes the warning.
|
||||
3. A legitimate cross-source forward/reverse pair passes.
|
||||
4. Two same-slug destination pages are not collapsed into one target identity.
|
||||
|
||||
The first assertion must fail against the pre-fix implementation.
|
||||
|
||||
### Engine contract tests
|
||||
|
||||
For PGLite and PostgreSQL:
|
||||
|
||||
1. Assert link rows expose exact from/to source IDs.
|
||||
2. Assert scalar reads still return explicit cross-source destinations.
|
||||
3. Assert federated reads still exclude out-of-grant endpoints.
|
||||
4. Assert `sourceIds` still takes precedence over scalar `sourceId`.
|
||||
5. Assert origin source identity is null when the origin is redacted by the federated branch.
|
||||
|
||||
### Parity and focused verification
|
||||
|
||||
Run:
|
||||
|
||||
- the focused backlink validator test;
|
||||
- source-isolation and federated link tests;
|
||||
- the Postgres/PGLite parity fixture with a test database;
|
||||
- related writer/post-write tests;
|
||||
- `bun run typecheck`.
|
||||
|
||||
Capture complete command output to files before inspecting summaries. Do not use production databases or restart the live service.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The `Link` change is additive at runtime. Existing consumers that read only slug or provenance fields continue to work. TypeScript object literals typed as complete `Link` values may need source fields; if compatibility pressure is high, the source fields can initially be optional in the public type while engine implementations and validator tests require their presence. The preferred contract is required endpoint source IDs because every persisted link always has both pages and therefore both source IDs.
|
||||
|
||||
No schema migration is required because source IDs already live on the joined `pages` rows.
|
||||
|
||||
## Operational constraints
|
||||
|
||||
The implementation phase does not deploy, restart GBrain, run production migrations, or alter the atomic conditional-write branch. Release, migration, and restart operations are a separate verified workflow and do not change this design's engine or validator semantics.
|
||||
@@ -216,19 +216,6 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
@@ -1019,13 +1006,6 @@ If you fetched this file by URL without cloning yet, the companion files live at
|
||||
|
||||
## Step 1: Install GBrain
|
||||
|
||||
> **NEVER install from the npm registry.** GBrain is not distributed on npm; the npm
|
||||
> package named `gbrain` is an unrelated package. Do NOT run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` (note the missing `github:` prefix — that's the trap). The only
|
||||
> supported sources are `github:garrytan/gbrain` and a git clone, exactly as shown below.
|
||||
> If an unrelated npm install is already present, remove it first
|
||||
> (`npm uninstall -g gbrain` / `bun remove -g gbrain`); `gbrain doctor` also detects this.
|
||||
|
||||
Default path (Bun is required — gbrain is a Bun + TypeScript runtime):
|
||||
|
||||
```bash
|
||||
@@ -1579,16 +1559,6 @@ This is the difference between a search engine and a brain. Search finds the pag
|
||||
|
||||
## Install
|
||||
|
||||
> [!WARNING]
|
||||
> **GBrain is NOT distributed on npm.** The npm package named `gbrain` is an unrelated
|
||||
> package with no connection to this project. Do not run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` — you'll get something else, and it can shadow the real binary on
|
||||
> your PATH. Install and upgrade ONLY via the documented paths below
|
||||
> (`bun install -g github:garrytan/gbrain`, or `git clone` + `bun install && bun link`).
|
||||
> If you already ran the npm install by mistake: `npm uninstall -g gbrain` /
|
||||
> `bun remove -g gbrain`, then reinstall from GitHub. `gbrain doctor` detects a
|
||||
> shadowing npm install and prints the fix.
|
||||
|
||||
GBrain is designed to be installed and operated by an AI agent. The fastest path is to have your agent do it for you. The CLI and MCP paths below are for people who want to wire it up themselves.
|
||||
|
||||
### Have your agent install it (recommended)
|
||||
|
||||
+33
-35
@@ -42,21 +42,20 @@
|
||||
"eval:autocut": "bun test test/search/autocut-eval.test.ts",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bash scripts/run-verify-parallel.sh",
|
||||
"check:source-config-leak": "bash scripts/check-source-config-leak.sh",
|
||||
"check:no-pii-agent-voice": "bash scripts/check-no-pii-in-agent-voice.sh",
|
||||
"check:synthetic-corpus-privacy": "bash scripts/check-synthetic-corpus-privacy.sh",
|
||||
"check:system-of-record": "bash scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "bash scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "bash scripts/check-cli-executable.sh",
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:all": "bash scripts/check-privacy.sh && bash scripts/check-proposal-pii.sh && bash scripts/check-test-real-names.sh && bash scripts/check-jsonb-pattern.sh && bash scripts/check-source-id-projection.sh && bash scripts/check-source-config-leak.sh && bash scripts/check-progress-to-stdout.sh && bash scripts/check-no-tracked-symlinks.sh && bash scripts/check-no-legacy-getconnection.sh && bash scripts/check-test-isolation.sh && bash scripts/check-trailing-newline.sh && bash scripts/check-wasm-embedded.sh && bash scripts/check-exports-count.sh && bash scripts/check-admin-build.sh && bash scripts/check-admin-scope-drift.sh && bash scripts/check-cli-executable.sh && bash scripts/check-skill-brain-first.sh && bash scripts/check-operations-filter-bypass.sh && bash scripts/check-gateway-routed-no-direct-anthropic.sh && bash scripts/check-worker-pool-atomicity.sh && bash scripts/check-key-files-current-state.sh && bash scripts/check-no-double-retry.sh && bash scripts/check-batch-audit-site.sh && bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:gateway-routed": "bash scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "bash scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "bash scripts/check-key-files-current-state.sh",
|
||||
"check:source-config-leak": "scripts/check-source-config-leak.sh",
|
||||
"check:no-pii-agent-voice": "scripts/check-no-pii-in-agent-voice.sh",
|
||||
"check:synthetic-corpus-privacy": "scripts/check-synthetic-corpus-privacy.sh",
|
||||
"check:system-of-record": "scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "scripts/check-cli-executable.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-proposal-pii.sh && scripts/check-test-real-names.sh && scripts/check-jsonb-pattern.sh && scripts/check-source-id-projection.sh && scripts/check-source-config-leak.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh && scripts/check-skill-brain-first.sh && scripts/check-operations-filter-bypass.sh && scripts/check-gateway-routed-no-direct-anthropic.sh && scripts/check-worker-pool-atomicity.sh && scripts/check-key-files-current-state.sh && scripts/check-no-double-retry.sh && scripts/check-batch-audit-site.sh",
|
||||
"check:gateway-routed": "scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "scripts/check-key-files-current-state.sh",
|
||||
"check:resolver": "bun src/cli.ts check-resolvable --strict --skills-dir skills/",
|
||||
"check:skill-brain-first": "bash scripts/check-skill-brain-first.sh",
|
||||
"check:wasm": "bash scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "bash scripts/check-trailing-newline.sh",
|
||||
"check:skill-brain-first": "scripts/check-skill-brain-first.sh",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:heavy": "bash scripts/run-heavy.sh",
|
||||
@@ -66,27 +65,26 @@
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check:jsonb": "bash scripts/check-jsonb-pattern.sh",
|
||||
"check:search-path": "bash scripts/check-search-path.sh",
|
||||
"check:no-double-retry": "bash scripts/check-no-double-retry.sh",
|
||||
"check:batch-audit-site": "bash scripts/check-batch-audit-site.sh",
|
||||
"check:worker-lock-renewal-shape": "bash scripts/check-worker-lock-renewal-shape.sh",
|
||||
"check:source-id-projection": "bash scripts/check-source-id-projection.sh",
|
||||
"check:privacy": "bash scripts/check-privacy.sh",
|
||||
"check:proposal-pii": "bash scripts/check-proposal-pii.sh",
|
||||
"check:eval-glossary": "bash scripts/check-eval-glossary-fresh.sh",
|
||||
"check:test-names": "bash scripts/check-test-real-names.sh",
|
||||
"check:progress": "bash scripts/check-progress-to-stdout.sh",
|
||||
"check:no-tracked-symlinks": "bash scripts/check-no-tracked-symlinks.sh",
|
||||
"check:exports-count": "bash scripts/check-exports-count.sh",
|
||||
"check:admin-build": "bash scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "bash scripts/check-admin-embedded.sh",
|
||||
"check:test-isolation": "bash scripts/check-test-isolation.sh",
|
||||
"check:fuzz-purity": "bash scripts/check-fuzz-purity.sh",
|
||||
"check:operations-filter-bypass": "bash scripts/check-operations-filter-bypass.sh",
|
||||
"check:fixture-privacy": "bash scripts/check-fixture-privacy.sh",
|
||||
"check:jsonb": "scripts/check-jsonb-pattern.sh",
|
||||
"check:search-path": "scripts/check-search-path.sh",
|
||||
"check:no-double-retry": "scripts/check-no-double-retry.sh",
|
||||
"check:batch-audit-site": "scripts/check-batch-audit-site.sh",
|
||||
"check:worker-lock-renewal-shape": "scripts/check-worker-lock-renewal-shape.sh",
|
||||
"check:source-id-projection": "scripts/check-source-id-projection.sh",
|
||||
"check:privacy": "scripts/check-privacy.sh",
|
||||
"check:proposal-pii": "scripts/check-proposal-pii.sh",
|
||||
"check:eval-glossary": "scripts/check-eval-glossary-fresh.sh",
|
||||
"check:test-names": "scripts/check-test-real-names.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "scripts/check-admin-embedded.sh",
|
||||
"check:test-isolation": "scripts/check-test-isolation.sh",
|
||||
"check:fuzz-purity": "scripts/check-fuzz-purity.sh",
|
||||
"check:operations-filter-bypass": "scripts/check-operations-filter-bypass.sh",
|
||||
"check:fixture-privacy": "scripts/check-fixture-privacy.sh",
|
||||
"check:conversation-parser": "bun src/cli.ts eval conversation-parser test/fixtures/conversation-formats/all.jsonl --no-llm",
|
||||
"check:source-scope-onboard": "bash scripts/check-source-scope-onboard.sh",
|
||||
"check:source-scope-onboard": "scripts/check-source-scope-onboard.sh",
|
||||
"postinstall": "bun run scripts/postinstall.ts",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
@@ -147,7 +145,7 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.42.68.1",
|
||||
"version": "0.42.66.1",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.4",
|
||||
|
||||
+7
-15
@@ -1,12 +1,12 @@
|
||||
---
|
||||
id: x-to-brain
|
||||
name: X-to-Brain
|
||||
version: 0.8.3
|
||||
version: 0.8.2
|
||||
description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts.
|
||||
category: sense
|
||||
requires: []
|
||||
secrets:
|
||||
- name: X_API_BEARER_TOKEN
|
||||
- name: X_BEARER_TOKEN
|
||||
description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search)
|
||||
where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens"
|
||||
- name: X_HANDLE
|
||||
@@ -16,7 +16,7 @@ health_checks:
|
||||
- type: http
|
||||
url: "https://api.x.com/2/users/by/username/$X_HANDLE"
|
||||
auth: bearer
|
||||
auth_token: "$X_API_BEARER_TOKEN"
|
||||
auth_token: "$X_BEARER_TOKEN"
|
||||
label: "X API"
|
||||
setup_time: 15 min
|
||||
cost_estimate: "$0-200/mo (Free tier: 1 app, read-only. Basic: $200/mo for search + higher limits)"
|
||||
@@ -118,11 +118,11 @@ Tell the user:
|
||||
Note: Free tier gives read-only access with low limits. Basic tier ($200/mo)
|
||||
gives search/recent endpoint and higher limits. Pro tier gets full archive search."
|
||||
|
||||
Set both `X_API_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
(app-only bearer tokens cannot call `/users/me` — that endpoint requires
|
||||
user-context OAuth — so validation uses the by-username lookup):
|
||||
```bash
|
||||
curl -sf -H "Authorization: Bearer $X_API_BEARER_TOKEN" \
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" \
|
||||
&& echo "PASS: X API connected" \
|
||||
|| echo "FAIL: X API token invalid"
|
||||
@@ -138,7 +138,7 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme
|
||||
|
||||
```bash
|
||||
# Look up the user's X user ID from their handle
|
||||
curl -sf -H "Authorization: Bearer $X_API_BEARER_TOKEN" \
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"'
|
||||
```
|
||||
|
||||
@@ -210,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/x-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.3","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
## Production Patterns (v0.8.1)
|
||||
@@ -438,14 +438,6 @@ Free tier works for personal monitoring. Basic tier needed for keyword search.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Upgrading from recipe v0.8.2 or earlier (token shows [missing] after upgrade):**
|
||||
- Older versions of this recipe named the token `X_BEARER_TOKEN`. The canonical
|
||||
name is `X_API_BEARER_TOKEN` — the name the built-in `x_handle_to_tweet`
|
||||
resolver reads. Rename the variable wherever you set it (shell profile, cron
|
||||
environment, `.env`) — same value, new name. A collector installed under the
|
||||
old name keeps running either way; the rename is what makes the integrations
|
||||
dashboard and the resolver see the token.
|
||||
|
||||
**API returns 403:**
|
||||
- Check your app has the right access level (Read or Read+Write)
|
||||
- Free tier apps can only use basic endpoints
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: every `bun test` invocation in workflows and runner scripts must
|
||||
# pass an explicit --timeout.
|
||||
#
|
||||
# Why: bun ignores bunfig.toml's `timeout` key (verified on 1.3.14), so a bare
|
||||
# `bun test` gets the 5000ms default for BOTH tests and beforeAll/beforeEach/
|
||||
# afterAll/afterEach hooks. Hooks do NOT inherit a test's third-arg timeout —
|
||||
# a file whose tests all declare `}, 30_000)` still has a 5s hook budget, and
|
||||
# slow setup (Postgres connect + migrations, PGLite cold start) flakes on
|
||||
# loaded CI runners with the signature `(unnamed) [5001ms] ... hook timed out`
|
||||
# (the #3545 jsonb-parity failure). The CLI --timeout flag is the one measured
|
||||
# mechanism that raises the hook budget uniformly; per-hook second-arg
|
||||
# timeouts work too but don't scale to ~400 slow hooks.
|
||||
#
|
||||
# Usage: scripts/check-bun-test-timeout.sh
|
||||
# Exit: 0 when clean, 1 when a bare `bun test` invocation is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Match executable `bun test` invocations. Exclude comment lines (#, //, *)
|
||||
# and lines that already carry --timeout anywhere.
|
||||
# Scope: workflows + runner scripts (the surfaces CI executes). package.json
|
||||
# script bodies route through scripts/ already; editing it is out of scope here.
|
||||
violations="$(grep -rnE '\bbun test\b' .github/workflows scripts 2>/dev/null \
|
||||
| grep -v -- '--timeout' \
|
||||
| grep -vE ':[[:space:]]*(#|//|\*)' \
|
||||
| grep -v 'check-bun-test-timeout' \
|
||||
|| true)"
|
||||
|
||||
if [ -n "$violations" ]; then
|
||||
echo "FAIL: bare 'bun test' without --timeout (5s default kills slow setup hooks):" >&2
|
||||
echo "$violations" >&2
|
||||
echo "" >&2
|
||||
echo "Add --timeout=60000 (see scripts/run-unit-shard.sh for the convention)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: every bun test invocation passes an explicit --timeout."
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Engine-live paths use static imports by default. A line-level
|
||||
# `engine-dynamic-import-ok` marker is required for a justified lazy import.
|
||||
#
|
||||
# Historical Windows runs associated imports on these paths with abrupt Bun
|
||||
# test-process exits, but system-wide commit exhaustion remained a confound.
|
||||
# This guard therefore enforces a reviewed engine-path hardening invariant; it
|
||||
# does not claim every dynamic import deterministically crashes Windows.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/check-engine-dynamic-import.sh
|
||||
# bash scripts/check-engine-dynamic-import.sh FILE [FILE...]
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" || exit 1
|
||||
|
||||
if [ "$#" -gt 0 ]; then
|
||||
FILES=("$@")
|
||||
else
|
||||
ROOT="$(git -C "$SCRIPT_DIR/.." rev-parse --show-toplevel 2>/dev/null || true)"
|
||||
[ -n "$ROOT" ] || ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$ROOT" || exit 1
|
||||
FILES=(
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
src/core/migrate.ts
|
||||
)
|
||||
fi
|
||||
|
||||
exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"
|
||||
@@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import ts from 'typescript';
|
||||
|
||||
const MARKER = 'engine-dynamic-import-ok';
|
||||
const MARKER_TOKEN_CHAR = /[\p{ID_Continue}$-]/u;
|
||||
const files = process.argv.slice(2);
|
||||
const violations: string[] = [];
|
||||
const readErrors: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
let sourceText: string;
|
||||
try {
|
||||
sourceText = await readFile(file, 'utf8');
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
readErrors.push(`ERROR: cannot read input file ${file}: ${detail}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceFile = ts.createSourceFile(
|
||||
file,
|
||||
sourceText,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
const lines = sourceText.split(/\r?\n/);
|
||||
const markerLines = new Set<number>();
|
||||
|
||||
if (sourceFile.parseDiagnostics.length > 0) {
|
||||
const diagnostics = sourceFile.parseDiagnostics
|
||||
.map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, ' '))
|
||||
.join('; ');
|
||||
readErrors.push(`ERROR: cannot parse input file ${file}: ${diagnostics}`);
|
||||
}
|
||||
|
||||
for (let markerPos = sourceText.indexOf(MARKER); markerPos >= 0; markerPos = sourceText.indexOf(MARKER, markerPos + MARKER.length)) {
|
||||
const before = Array.from(sourceText.slice(0, markerPos)).at(-1);
|
||||
const after = Array.from(sourceText.slice(markerPos + MARKER.length))[0];
|
||||
const standaloneMarker = (!before || !MARKER_TOKEN_CHAR.test(before))
|
||||
&& (!after || !MARKER_TOKEN_CHAR.test(after));
|
||||
const token = ts.getTokenAtPosition(sourceFile, markerPos);
|
||||
const insideToken = token.getStart(sourceFile) <= markerPos && markerPos < token.end;
|
||||
if (standaloneMarker && !insideToken) {
|
||||
markerLines.add(sourceFile.getLineAndCharacterOfPosition(markerPos).line);
|
||||
}
|
||||
}
|
||||
|
||||
function visit(node: ts.Node): void {
|
||||
if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) {
|
||||
const { line } = sourceFile.getLineAndCharacterOfPosition(node.expression.getStart(sourceFile));
|
||||
const sourceLine = lines[line] ?? '';
|
||||
if (!markerLines.has(line)) {
|
||||
violations.push(` ${file}:${line + 1}:${sourceLine}`);
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
|
||||
visit(sourceFile);
|
||||
}
|
||||
|
||||
for (const error of readErrors) console.error(error);
|
||||
|
||||
if (violations.length > 0) {
|
||||
console.error('ERROR: unreviewed dynamic import on an engine-live path:');
|
||||
console.error();
|
||||
console.error(violations.join('\n'));
|
||||
console.error();
|
||||
console.error('Prefer a static top-level import. If lazy loading is load-bearing,');
|
||||
console.error("append 'engine-dynamic-import-ok' to that exact line and document");
|
||||
console.error('the startup or soft-failure boundary that requires it.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (readErrors.length > 0) process.exit(1);
|
||||
|
||||
console.log(`check-engine-dynamic-import: ok (${files.length} file(s) scanned)`);
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any symlink is tracked in git.
|
||||
#
|
||||
# A symlink committed from a build sandbox points at a path that exists on
|
||||
# exactly one machine. Everywhere else the checkout produces a dangling
|
||||
# link, and anything that opens it fails. That is not hypothetical: commit
|
||||
# faf5cdba landed `node_modules -> /tmp/fleet/repo/node_modules`, which made
|
||||
# `bun install` abort with `ENOENT: could not open the "node_modules"
|
||||
# directory` on every fresh clone, and took `gbrain upgrade`'s bun-link path
|
||||
# down with it (the auto-upgrade runs `bun install`, so the printed manual
|
||||
# fallback failed the same way).
|
||||
#
|
||||
# .gitignore alone does not prevent this. A `node_modules/` pattern with a
|
||||
# trailing slash matches directories ONLY, so a symlink of the same name is
|
||||
# never ignored. Dropping the slash closes that hole, but `git add -f` still
|
||||
# walks straight past it. This guard is the backstop.
|
||||
#
|
||||
# The repo has no legitimate tracked symlinks, so the allowlist starts
|
||||
# empty. If you ever need one, add its exact repo-relative path to ALLOWLIST
|
||||
# below and explain why — a relative link that resolves inside the repo is
|
||||
# defensible; an absolute one almost never is.
|
||||
#
|
||||
# Usage: scripts/check-no-tracked-symlinks.sh
|
||||
# Exit: 0 when clean, 1 when a tracked symlink is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Paths permitted to be tracked symlinks. Empty by design.
|
||||
ALLOWLIST=()
|
||||
|
||||
# Git records symlinks with mode 120000. Field 4 of `ls-files -s` is the path
|
||||
# (tab-separated from the stage number), so cut on the tab to keep paths with
|
||||
# spaces intact.
|
||||
found="$(git ls-files -s | awk '$1 == "120000"' | cut -f2- || true)"
|
||||
|
||||
if [ -n "$found" ]; then
|
||||
filtered="$found"
|
||||
for f in "${ALLOWLIST[@]:-}"; do
|
||||
[ -z "$f" ] && continue
|
||||
filtered="$(echo "$filtered" | grep -vxF "$f" || true)"
|
||||
done
|
||||
|
||||
if [ -n "$filtered" ]; then
|
||||
echo "ERROR: symlink(s) tracked in git:"
|
||||
echo
|
||||
while IFS= read -r path; do
|
||||
[ -z "$path" ] && continue
|
||||
target="$(git cat-file blob ":$path" 2>/dev/null || echo '<unreadable>')"
|
||||
echo " $path -> $target"
|
||||
done <<< "$filtered"
|
||||
echo
|
||||
echo "A committed symlink resolves on the machine that created it and"
|
||||
echo "nowhere else. Untrack it:"
|
||||
echo
|
||||
echo " git rm --cached <path>"
|
||||
echo
|
||||
echo "If the path is build output (node_modules, dist, bin), also confirm"
|
||||
echo "it is covered by .gitignore WITHOUT a trailing slash — a trailing"
|
||||
echo "slash matches directories only and lets the symlink through."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "check-no-tracked-symlinks: OK (no tracked symlinks)"
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Import an envelope-v0 file (a JSON serialization of AI chat history; format
|
||||
* spec: github.com/memvelope/memvelope) into a brain repo as one Markdown page
|
||||
* per conversation, which `gbrain sync` ingests.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/envelope-to-gbrain.mjs <envelope.mve.json> [outDir]
|
||||
*
|
||||
* Zero dependencies. Deterministic. No network. It does NOT call gbrain — it
|
||||
* only writes Markdown files.
|
||||
*
|
||||
* Output layout:
|
||||
* - One page per conversation, filename = date + conversation id (shared
|
||||
* titles cannot collide; the id is the natural key). A duplicate id
|
||||
* overwrites its own filename and warns on stderr; stdout reports DISTINCT
|
||||
* files written, not write calls.
|
||||
* - Frontmatter: `type: conversation` (keeps pages eligible for
|
||||
* conversation-facts extraction and chronicle behavior after sync), the
|
||||
* source provider, the conversation id, and `origin: memvelope/envelope-v0`.
|
||||
* - Page `date` is the first 10 chars of the conversation's ISO-8601
|
||||
* `created_at`. Body keeps message-id citations beside each speaker turn.
|
||||
*
|
||||
* Memory: the whole envelope is held in memory (no streaming); envelopes are
|
||||
* far smaller than the vendor exports they serialize.
|
||||
*
|
||||
* Verify:
|
||||
* node scripts/envelope-to-gbrain.mjs test/fixtures/memvelope/sample.mve.json /tmp/out
|
||||
* -> expect "wrote 1 markdown page(s)"
|
||||
* bun test test/envelope-to-gbrain.test.ts
|
||||
*
|
||||
* STATUS: live-verified against gbrain v0.42.56.0 on 2026-07-03: the sample
|
||||
* fixture -> 1 page; a real 662MB Claude export -> 353 conversations = 353
|
||||
* distinct pages (no collisions), searchable after sync with provenance and
|
||||
* message-id citations intact.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const [, , envelopePath, outDir = './brain/conversations'] = process.argv;
|
||||
if (!envelopePath) {
|
||||
console.error('usage: node envelope-to-gbrain.mjs <envelope.mve.json> [outDir]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const env = JSON.parse(readFileSync(envelopePath, 'utf8'));
|
||||
if (env.memvelope !== 'envelope-v0') {
|
||||
console.error(`not an envelope-v0 file (memvelope field = ${JSON.stringify(env.memvelope)})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const slug = (s, fallback) =>
|
||||
(String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback).slice(0, 60);
|
||||
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
const filesWritten = new Set();
|
||||
let collisions = 0;
|
||||
const conversations = env.conversations || [];
|
||||
for (const [i, c] of conversations.entries()) {
|
||||
const date = (c.created_at || '').slice(0, 10);
|
||||
// Name the file by the conversation's own id — the natural unique key — so two
|
||||
// conversations that share a date and title can never silently overwrite each
|
||||
// other. The date only leads as a human/chronological sort prefix; the id
|
||||
// carries uniqueness. Positional fallback keeps names unique and deterministic
|
||||
// when an envelope omits an id.
|
||||
// One predicate for "this conversation carries its own id", shared by the
|
||||
// filename and the frontmatter below. Keeping it in a single place is what
|
||||
// stops the two from disagreeing about whether an id exists.
|
||||
const hasId = typeof c.id === 'string' && c.id.trim() !== '';
|
||||
const convId = hasId ? c.id.trim() : `conv-${i + 1}`;
|
||||
const name = `${date || '0000-00-00'}-${slug(convId, `conv-${i + 1}`)}.md`;
|
||||
// gbrain reads YAML frontmatter + markdown body; keep provenance in frontmatter.
|
||||
// Emit `type: conversation` so gbrain stores these as conversation pages rather
|
||||
// than defaulting to the generic `concept`. gbrain is open-typed — it takes an
|
||||
// explicit frontmatter `type` verbatim — and its conversation-aware features
|
||||
// (conversation-facts extraction, the conversation_format_coverage check,
|
||||
// chronicle eligibility) key off `type == 'conversation'`.
|
||||
const front = [
|
||||
'---',
|
||||
'type: conversation',
|
||||
`title: ${JSON.stringify(c.title || 'Untitled conversation')}`,
|
||||
`date: ${date || 'null'}`,
|
||||
// Every interpolated value is quoted. An envelope is a third-party file, so
|
||||
// a provider string carrying a newline would otherwise close this scalar and
|
||||
// inject arbitrary frontmatter keys into the page gbrain ingests.
|
||||
`source: ${JSON.stringify(env.meta?.source_provider || 'unknown')}`,
|
||||
// Omit the key entirely when the envelope carries no id, rather than
|
||||
// emitting the literal `undefined` or a synthesized `conv-N` — the positional
|
||||
// fallback names the file, but it is not a memvelope conversation id and
|
||||
// must not be recorded as one.
|
||||
...(hasId ? [`memvelope_conversation_id: ${JSON.stringify(convId)}`] : []),
|
||||
'origin: memvelope/envelope-v0',
|
||||
'---',
|
||||
'',
|
||||
].join('\n');
|
||||
const body = (c.messages || [])
|
||||
.map((m) => `**${m.role === 'user' ? 'Me' : 'Assistant'}** (${m.ts || 'no timestamp'} · ${m.id}):\n\n${m.text}`)
|
||||
.join('\n\n---\n\n');
|
||||
// Never lose a page silently: if two conversations still map to the same
|
||||
// filename (e.g. an envelope carrying duplicate ids), warn loudly instead of
|
||||
// overwriting in silence, and report the count of DISTINCT files written — not
|
||||
// the number of write calls, which is what hid the old title-collision bug.
|
||||
if (filesWritten.has(name)) {
|
||||
collisions += 1;
|
||||
console.warn(`warning: filename collision on "${name}" — conversation id ${JSON.stringify(c.id)} is not unique; overwriting the earlier page.`);
|
||||
}
|
||||
writeFileSync(join(outDir, name), front + `# ${c.title || 'Conversation'}\n\n` + body + '\n');
|
||||
filesWritten.add(name);
|
||||
}
|
||||
console.log(`wrote ${filesWritten.size} markdown page(s) to ${outDir} — point gbrain's sync at this directory.`);
|
||||
if (collisions) {
|
||||
console.warn(`warning: ${collisions} filename collision(s) — ${collisions} page(s) overwritten. Deduplicate conversation ids in the envelope to avoid data loss.`);
|
||||
}
|
||||
+2
-3
@@ -162,9 +162,8 @@ for f in "${files[@]}"; do
|
||||
if [ -n "${DATABASE_URL:-}" ]; then
|
||||
psql "$DATABASE_URL" -At -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid != pg_backend_pid() AND datname = current_database()" >/dev/null 2>&1 || true
|
||||
fi
|
||||
# Hard outer timeout (180s per file). bun's --timeout covers tests AND
|
||||
# hooks (measured on 1.3.14), but it's timer-based: a PGLite WASM call
|
||||
# that blocks the event loop synchronously never lets the timer fire and
|
||||
# Hard outer timeout (180s per file). bun's --timeout is per-test; if a
|
||||
# PGLite WASM call hangs in beforeAll/afterAll, --timeout never fires and
|
||||
# the file wedges indefinitely. gtimeout/timeout SIGKILLs the file so the
|
||||
# suite advances. gtimeout (macOS via coreutils) preferred; timeout (Linux)
|
||||
# fallback; bare bun (no outer cap) if neither is installed.
|
||||
|
||||
@@ -42,7 +42,6 @@ CHECKS=(
|
||||
"check:source-id-projection"
|
||||
"check:source-config-leak"
|
||||
"check:progress"
|
||||
"check:no-tracked-symlinks"
|
||||
"check:test-isolation"
|
||||
"check:wasm"
|
||||
"check:admin-build"
|
||||
@@ -64,7 +63,6 @@ CHECKS=(
|
||||
"check:source-scope-onboard"
|
||||
"check:no-double-retry"
|
||||
"check:batch-audit-site"
|
||||
"check:engine-dynamic-import"
|
||||
"check:worker-lock-renewal-shape"
|
||||
"typecheck"
|
||||
)
|
||||
|
||||
@@ -139,9 +139,9 @@ edits writes a new receipt).
|
||||
|
||||
| Slot | Default | Provider |
|
||||
|------|---------|----------|
|
||||
| A | `openai:gpt-5.2` | OpenAI |
|
||||
| A | `openai:gpt-4o` | OpenAI |
|
||||
| B | `anthropic:claude-opus-4-7` | Anthropic |
|
||||
| C | `deepseek:deepseek-v4-pro` | DeepSeek |
|
||||
| C | `google:gemini-1.5-pro` | Google |
|
||||
|
||||
**These MUST be frontier models from DIFFERENT providers.** Using a single
|
||||
provider's family or budget models defeats the purpose — different families
|
||||
|
||||
+23
-144
@@ -9,7 +9,7 @@ installSigchldHandler();
|
||||
import { installSignalHandlers as installCleanupSignalHandlers } from './core/process-cleanup.ts';
|
||||
installCleanupSignalHandlers();
|
||||
|
||||
import { readFileSync, existsSync, unlinkSync, fstatSync } from 'fs';
|
||||
import { readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { spawn } from 'child_process';
|
||||
import {
|
||||
readUpdateCache,
|
||||
@@ -55,7 +55,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector', 'backfill']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -344,11 +344,6 @@ async function main() {
|
||||
// them out of the engine try/catch is safe and unlocks routing.
|
||||
const params = parseOpArgs(op, subArgs);
|
||||
|
||||
// #3513: stdin fill moved out of parseOpArgs so a non-TTY stdin with no
|
||||
// piped input can't block the parse forever — the bounded read leaves the
|
||||
// param unset on timeout and the required-param check below fails fast.
|
||||
await applyStdinParam(op, params);
|
||||
|
||||
// v0.27.1 (`gbrain query --image <path>`): swap the `image` param from
|
||||
// a filesystem path into base64 bytes + mime. The op accepts base64; the
|
||||
// CLI accepts a path. Helper is exported so tests can exercise the
|
||||
@@ -809,99 +804,18 @@ export function parseOpArgs(op: Operation, args: string[]): Record<string, unkno
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* #3513: read stdin into an op's stdin-capable param without ever blocking
|
||||
* forever. The old inline `readFileSync(0)` in parseOpArgs assumed non-TTY
|
||||
* implies piped content; a non-TTY stdin with NO input (CI step, cron job,
|
||||
* agent harness holding an unwritten pipe open) blocked the read until kill.
|
||||
*
|
||||
* Strategy by fd kind (fstat):
|
||||
* - TTY: skip, as before (interactive input is not an op-param source).
|
||||
* - regular file / /dev/null / anything not a pipe or socket: readFileSync
|
||||
* returns without blocking (`gbrain put x < file`, `< /dev/null` → '').
|
||||
* - FIFO/socket: stream-read with a deadline on the FIRST byte only. A real
|
||||
* pipe (`echo foo | gbrain put x`, heredocs) delivers its first byte
|
||||
* within milliseconds; once any data arrives the deadline is lifted and
|
||||
* we read to EOF like readFileSync did (slow producers stay supported).
|
||||
* An empty-but-closed pipe (`: | gbrain put x`) EOFs immediately → ''.
|
||||
* A pipe that never delivers a byte times out → param stays unset, so
|
||||
* the existing required-param usage error fires (fail fast, exit 1).
|
||||
*
|
||||
* GBRAIN_STDIN_TIMEOUT_MS overrides the first-byte deadline (default 5000).
|
||||
* Exported for tests; called by the op dispatch right after parseOpArgs.
|
||||
*/
|
||||
export async function applyStdinParam(
|
||||
op: Operation,
|
||||
params: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
// Branch shape (stdin hint + missing param + `!process.stdin.isTTY` gate +
|
||||
// 5MB cap) is pinned by the R4 regression test for PR #1325's Windows fix
|
||||
// (test/cycle/regression-pr-wave-r1-r2-r4.test.ts) — keep the spelling.
|
||||
// Read stdin for content params
|
||||
if (op.cliHints?.stdin && !params[op.cliHints.stdin] && !process.stdin.isTTY) {
|
||||
const content = await readStdinBounded();
|
||||
if (content === null) return; // no input arrived — let the required-param check fail fast
|
||||
const stdinContent = readFileSync(0, 'utf-8');
|
||||
const MAX_STDIN = 5_000_000; // 5MB
|
||||
if (Buffer.byteLength(content, 'utf-8') > MAX_STDIN) {
|
||||
if (Buffer.byteLength(stdinContent, 'utf-8') > MAX_STDIN) {
|
||||
console.error(`Error: stdin content exceeds ${MAX_STDIN} bytes. Split into smaller inputs.`);
|
||||
process.exit(1);
|
||||
}
|
||||
params[op.cliHints.stdin] = content;
|
||||
params[op.cliHints.stdin] = stdinContent;
|
||||
}
|
||||
}
|
||||
|
||||
/** First-byte deadline for pipe/socket stdin (#3513). Env-overridable escape hatch. */
|
||||
function stdinFirstByteTimeoutMs(): number {
|
||||
const n = Number(process.env.GBRAIN_STDIN_TIMEOUT_MS);
|
||||
return Number.isFinite(n) && n > 0 ? n : 5000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the full stdin content, '' for a readable-but-empty stdin, or
|
||||
* null when stdin is a pipe/socket that never delivered a byte within the
|
||||
* first-byte deadline (or the fd is closed/unreadable).
|
||||
*/
|
||||
export async function readStdinBounded(): Promise<string | null> {
|
||||
let isPipeOrSocket: boolean;
|
||||
try {
|
||||
const st = fstatSync(0);
|
||||
isPipeOrSocket = st.isFIFO() || st.isSocket();
|
||||
} catch {
|
||||
return null; // closed/invalid fd — treat as no input
|
||||
}
|
||||
if (!isPipeOrSocket) {
|
||||
// Regular file redirect, /dev/null, etc. — read returns without blocking.
|
||||
try {
|
||||
return readFileSync(0, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return await new Promise<string | null>((resolve) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let gotData = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (!gotData) {
|
||||
process.stdin.destroy();
|
||||
resolve(null);
|
||||
}
|
||||
}, stdinFirstByteTimeoutMs());
|
||||
const finish = () => {
|
||||
clearTimeout(timer);
|
||||
resolve(Buffer.concat(chunks).toString('utf-8'));
|
||||
};
|
||||
process.stdin.on('data', (c: Buffer) => {
|
||||
if (!gotData) {
|
||||
gotData = true;
|
||||
clearTimeout(timer); // deadline applies to the FIRST byte only
|
||||
}
|
||||
chunks.push(c);
|
||||
});
|
||||
process.stdin.once('end', finish);
|
||||
process.stdin.once('error', finish);
|
||||
});
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -958,8 +872,7 @@ export function applyThinClientSourceScope(
|
||||
params.source_id = resolved;
|
||||
}
|
||||
|
||||
// Exported for tests (same import-safety contract as applyThinClientSourceScope).
|
||||
export async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
|
||||
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
@@ -971,21 +884,16 @@ export async function makeContext(engine: BrainEngine, params: Record<string, un
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
try {
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
} catch (err) {
|
||||
// #1712: an EXPLICIT --source that fails to resolve (invalid id, or a
|
||||
// source that doesn't exist) must error loudly — the blanket swallow
|
||||
// turned `--source __all__` and typos into a silent `default` scope,
|
||||
// which is how three bug reports became debugging sessions.
|
||||
if (explicit) throw err;
|
||||
// Ambient resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
// to the cross-source view (D16 back-compat path).
|
||||
sourceId = undefined;
|
||||
@@ -1707,12 +1615,14 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
// Per-command default: search 30s, sources list 10s. User --timeout=Ns wins.
|
||||
// Other commands (import, embed, doctor, etc.) keep their existing
|
||||
// unbounded connect — destructive / long-running commands shouldn't get
|
||||
// a default kill switch. The gate below is per-command (#3013): only the
|
||||
// commands dispatchReadOnlyCommand handles may enter this path — a
|
||||
// user-supplied --timeout on a write command must never reroute it here.
|
||||
// a default kill switch.
|
||||
const readOnlyDefaultTimeoutMs =
|
||||
command === 'search' ? 30_000 :
|
||||
command === 'sources' && (args[0] === 'list' || args[0] === undefined) ? 10_000 :
|
||||
null;
|
||||
const cliOptsResolved = getCliOptions();
|
||||
const userTimeoutMs = cliOptsResolved.timeoutMs;
|
||||
const readOnlyTimeoutMs = resolveReadOnlyDispatchTimeoutMs(command, args, userTimeoutMs);
|
||||
const readOnlyTimeoutMs = userTimeoutMs ?? readOnlyDefaultTimeoutMs;
|
||||
|
||||
if (readOnlyTimeoutMs !== null) {
|
||||
const { withTimeout, OperationTimeoutError } = await import('./core/timeout.ts');
|
||||
@@ -2251,24 +2161,16 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
//
|
||||
// v0.30.1: still works; canonical entrypoint is now `gbrain backfill
|
||||
// effective_date`. This command stays as a thin alias for back-compat.
|
||||
//
|
||||
// #1963: pass the already-connected engine. The command used to build
|
||||
// + connect its OWN engine here, which self-deadlocked on the PGLite
|
||||
// data-dir lock (this process already holds it via connectEngine
|
||||
// above) — 30s spin, then exit 1, on every PGLite invocation.
|
||||
const { reindexFrontmatterCli } = await import('./commands/reindex-frontmatter.ts');
|
||||
await reindexFrontmatterCli(engine, args);
|
||||
break;
|
||||
await reindexFrontmatterCli(args);
|
||||
return; // reindexFrontmatterCli handles its own engine lifecycle
|
||||
}
|
||||
case 'backfill': {
|
||||
// v0.30.1: first-class generic backfill command. Subcommand dispatch
|
||||
// is inside runBackfillCommand (kind | list | --help).
|
||||
// #1963: same double-connect class as reindex-frontmatter — reuse the
|
||||
// connected engine instead of building a second one on the same
|
||||
// PGLite data dir.
|
||||
const { runBackfillCommand } = await import('./commands/backfill.ts');
|
||||
await runBackfillCommand(engine, args);
|
||||
break;
|
||||
await runBackfillCommand(args);
|
||||
return;
|
||||
}
|
||||
case 'code-callers': {
|
||||
// v0.20.0 Cathedral II Layer 10 (C4): "who calls <symbol>?"
|
||||
@@ -2311,28 +2213,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #3013: decide whether an invocation enters the read-only connect+dispatch
|
||||
* timeout path, and with what wallclock. Returns null for every command
|
||||
* dispatchReadOnlyCommand can't handle. The gate used to be "a timeout is
|
||||
* present" — so a user-supplied --timeout on a write command (`sync`,
|
||||
* `embed`, `import`, ...) hijacked dispatch into the read-only path, which
|
||||
* threw and exited 1 before any work ran. Pure; exported for the
|
||||
* regression test.
|
||||
*/
|
||||
export function resolveReadOnlyDispatchTimeoutMs(
|
||||
command: string,
|
||||
subArgs: string[],
|
||||
userTimeoutMs: number | null,
|
||||
): number | null {
|
||||
if (command !== 'search' && command !== 'sources') return null;
|
||||
const defaultMs =
|
||||
command === 'search' ? 30_000 :
|
||||
(subArgs[0] === 'list' || subArgs[0] === undefined) ? 10_000 :
|
||||
null;
|
||||
return userTimeoutMs ?? defaultMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.41.6.0 D3: dispatch helper for the read-only commands that take a
|
||||
* default wallclock timeout (`gbrain search`, `gbrain sources list`).
|
||||
@@ -2568,7 +2448,6 @@ TOOLS
|
||||
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
|
||||
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
|
||||
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
|
||||
backfill <kind|list> v0.30.1: run a registered backfill (effective-date, ...)
|
||||
orphans [--json] [--count] Find pages with no inbound wikilinks
|
||||
salience [--days N] [--kind P] v0.29: pages ranked by emotional + activity salience
|
||||
anomalies [--since D] [--sigma N] v0.29: cohort-based statistical anomalies (tag, type)
|
||||
|
||||
@@ -1,42 +1,9 @@
|
||||
import { defaultTimeoutMsFor } from '../core/minions/handler-timeouts.ts';
|
||||
|
||||
// #2781: the full-cycle floor used to be a literal `1_800_000` that merely
|
||||
// HAPPENED to match the 'autopilot-cycle' / 'autopilot-global-maintenance'
|
||||
// handler anchors (`HANDLER_DEFAULT_TIMEOUT_MS`, #1737) instead of being
|
||||
// derived from them. A duplicated literal can silently drift from the
|
||||
// handler default it's supposed to track — which is exactly the bug class
|
||||
// #2781 reported (an explicit `timeout_ms` stamp permanently overrides the
|
||||
// handler default per `queue.ts`'s `opts?.timeout_ms ?? defaultTimeoutMsFor`,
|
||||
// so a stale/lower literal here would starve a phase the handler default
|
||||
// was sized for). Deriving the floor from `defaultTimeoutMsFor` for both
|
||||
// full-cycle job names keeps the stamp coupled to its anchor by construction.
|
||||
// Fail fast (not `?? 0`) if either handler ever loses its entry in
|
||||
// HANDLER_DEFAULT_TIMEOUT_MS — silently falling back to "no floor" would
|
||||
// reintroduce #2781 rather than surface the drift.
|
||||
function requireHandlerAnchorMs(jobName: string): number {
|
||||
const ms = defaultTimeoutMsFor(jobName);
|
||||
if (ms === null) {
|
||||
throw new Error(
|
||||
`resolveAutopilotDispatchTimeoutMs: '${jobName}' has no entry in HANDLER_DEFAULT_TIMEOUT_MS ` +
|
||||
'(handler-timeouts.ts) — the full-cycle timeout floor can no longer be derived from it. ' +
|
||||
'See #2781: a missing/removed anchor here silently reintroduces the interval-derived stamp ' +
|
||||
'permanently overriding the handler default.',
|
||||
);
|
||||
}
|
||||
return ms;
|
||||
}
|
||||
|
||||
const FULL_CYCLE_TIMEOUT_FLOOR_MS = Math.max(
|
||||
requireHandlerAnchorMs('autopilot-cycle'),
|
||||
requireHandlerAnchorMs('autopilot-global-maintenance'),
|
||||
);
|
||||
|
||||
export function resolveAutopilotDispatchTimeoutMs(
|
||||
baseIntervalSeconds: number,
|
||||
fullCycle: boolean,
|
||||
): number {
|
||||
const intervalDerivedTimeoutMs = Math.max(baseIntervalSeconds * 2 * 1000, 300_000);
|
||||
return fullCycle
|
||||
? Math.max(intervalDerivedTimeoutMs, FULL_CYCLE_TIMEOUT_FLOOR_MS)
|
||||
? Math.max(intervalDerivedTimeoutMs, 1_800_000)
|
||||
: intervalDerivedTimeoutMs;
|
||||
}
|
||||
|
||||
@@ -981,21 +981,12 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// can't shrink throughput (codex #9/D5). autopilot-cycle jobs run on
|
||||
// the 'default' queue, so that's the concurrency we compare against.
|
||||
const fanoutMax = await resolveEffectiveFanoutMax(engine, 'default');
|
||||
// #2781: both 'autopilot-cycle' (per-source) and 'autopilot-global-
|
||||
// maintenance' carry a 30-min handler anchor (handler-timeouts.ts)
|
||||
// because a full cycle can outlive short daemon intervals — unlike
|
||||
// the lighter interval-derived `timeoutMs` above (sync/freshness,
|
||||
// extract-atoms-drain, targeted small-plan steps), which have no
|
||||
// such anchor and are meant to stay interval-derived. Naming this
|
||||
// separately (rather than reusing the outer `timeoutMs`) avoids
|
||||
// the #2781 bug class: dispatchGlobalMaintenance previously reused
|
||||
// the outer non-full-cycle `timeoutMs` by shorthand, silently
|
||||
// dropping its own handler anchor.
|
||||
const fullCycleTimeoutMs = resolveAutopilotDispatchTimeoutMs(baseInterval, true);
|
||||
const result = await dispatchPerSource(engine, queue, {
|
||||
repoPath,
|
||||
slot,
|
||||
timeoutMs: fullCycleTimeoutMs,
|
||||
// Full cycles can outlive short daemon intervals. Keep lighter dispatches
|
||||
// interval-derived while giving per-source consolidation enough time.
|
||||
timeoutMs: resolveAutopilotDispatchTimeoutMs(baseInterval, true),
|
||||
fanoutMax,
|
||||
jsonMode,
|
||||
});
|
||||
@@ -1006,7 +997,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// the per-source path (legacy single-source still runs everything).
|
||||
if (!result.legacy_fallback) {
|
||||
try {
|
||||
await dispatchGlobalMaintenance(engine, queue, { repoPath, slot, timeoutMs: fullCycleTimeoutMs, jsonMode });
|
||||
await dispatchGlobalMaintenance(engine, queue, { repoPath, slot, timeoutMs, jsonMode });
|
||||
} catch (e) {
|
||||
if (jsonMode) process.stderr.write(JSON.stringify({ event: 'global_maintenance_dispatch_failed', error: e instanceof Error ? e.message : String(e) }) + '\n');
|
||||
}
|
||||
|
||||
@@ -16,10 +16,10 @@
|
||||
* always reserving 1 connection for HNSW + heartbeat + doctor probes.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { resolveDirectPoolSize } from '../core/connection-manager.ts';
|
||||
import { listBackfills, getBackfill } from '../core/backfill-registry.ts';
|
||||
import { runBackfill, clearBackfillCheckpoint } from '../core/backfill-base.ts';
|
||||
import { loadConfig, toEngineConfig } from '../core/config.ts';
|
||||
|
||||
interface BackfillArgs {
|
||||
kind?: string;
|
||||
@@ -114,14 +114,7 @@ function clampConcurrency(requested: number | undefined): { effective: number; w
|
||||
return { effective: requested };
|
||||
}
|
||||
|
||||
/**
|
||||
* #1963 (same class as reindex-frontmatter): takes the ALREADY-CONNECTED
|
||||
* engine from cli.ts's dispatch. Building a second engine here deadlocked on
|
||||
* the PGLite data-dir lock (cli.ts's `connectEngine()` already holds it in
|
||||
* this same process) — every `gbrain backfill <kind>` on PGLite timed out
|
||||
* after 30s. Engine lifecycle belongs to cli.ts's connect + teardown.
|
||||
*/
|
||||
export async function runBackfillCommand(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
export async function runBackfillCommand(args: string[]): Promise<void> {
|
||||
const cli = parseArgs(args);
|
||||
if (cli.help) { printHelp(); return; }
|
||||
|
||||
@@ -151,10 +144,20 @@ export async function runBackfillCommand(engine: BrainEngine, args: string[]): P
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const config = loadConfig();
|
||||
if (!config) {
|
||||
console.error('No brain configured. Run: gbrain init');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// X5 admission control — clamp concurrency to direct-pool capacity.
|
||||
const { effective: concurrency, warning } = clampConcurrency(cli.concurrency);
|
||||
if (warning) console.warn(warning);
|
||||
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const engine = await createEngine(toEngineConfig(config));
|
||||
await engine.connect(toEngineConfig(config));
|
||||
|
||||
if (cli.fresh) {
|
||||
await clearBackfillCheckpoint(engine, reg.spec.name);
|
||||
console.log(`Cleared checkpoint for backfill.${reg.spec.name}`);
|
||||
@@ -189,6 +192,7 @@ export async function runBackfillCommand(engine: BrainEngine, args: string[]): P
|
||||
if (result.cappedByMaxRows) console.log(` ⚠️ Capped by --max-rows; more remain.`);
|
||||
if (result.cappedByErrors) console.log(` ⚠️ Capped by --max-errors at ${result.errors}.`);
|
||||
|
||||
await engine.disconnect();
|
||||
if (result.cappedByErrors) process.exit(1);
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
semverGt,
|
||||
semverLte,
|
||||
} from '../core/semver.ts';
|
||||
import { readUpdateCache, writeUpdateCache, type UpdateMarker } from '../core/self-upgrade.ts';
|
||||
import { writeUpdateCache, type UpdateMarker } from '../core/self-upgrade.ts';
|
||||
|
||||
/** Best-effort cache write — a read-only ~/.gbrain must never make the check throw. */
|
||||
function safeWriteCache(marker: UpdateMarker): void {
|
||||
@@ -45,53 +45,26 @@ function upgradeCommandForMethod(method: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
/** Where the latest version is resolved from. gbrain publishes NO GitHub
|
||||
* releases (the `releases/latest` API is a permanent 404), so the release
|
||||
* train's source of truth is the `VERSION` file on master — same trusted host
|
||||
* `fetchChangelog` already uses. An npm fallback was rejected: the `gbrain`
|
||||
* package on npm is an unrelated GPU library (#505), so it would produce false
|
||||
* upgrade prompts pointing at a stranger's package. */
|
||||
const VERSION_SOURCE_URL = 'https://raw.githubusercontent.com/garrytan/gbrain/master/VERSION';
|
||||
const RELEASE_NOTES_URL = 'https://github.com/garrytan/gbrain/blob/master/CHANGELOG.md';
|
||||
|
||||
/** Extract a version from the raw VERSION file body: first line, optional `v`
|
||||
* prefix, optional `-suffix` channel tag (`0.31.1.1-fixwave` compares as its
|
||||
* numeric base — fail-safe: a suffix-only bump never prompts). Body is bounded
|
||||
* before parsing so a malformed/huge response can't blow up the check. */
|
||||
export function parseVersionFileBody(body: string): string | null {
|
||||
const firstLine = body.slice(0, 256).trim().split('\n')[0].trim();
|
||||
const m = firstLine.match(/^v?(\d+\.\d+\.\d+(?:\.\d+)?)(?:[-+][0-9A-Za-z.-]+)?$/);
|
||||
return m && isValidVersionString(m[1]) ? m[1] : null;
|
||||
}
|
||||
|
||||
export type LatestReleaseResult =
|
||||
| { ok: true; tag: string; published_at: string; url: string }
|
||||
| { ok: false; reason: 'network_error' | 'no_releases' };
|
||||
|
||||
/**
|
||||
* Resolve the latest published gbrain version (from VERSION on master — see
|
||||
* VERSION_SOURCE_URL). Exported (v0.42) so the self-upgrade refresh path and
|
||||
* tests can reuse it. 5s timeout — this runs on the detached refresh, never the
|
||||
* hot path. Failures are discriminated: `network_error` (offline/timeout) vs
|
||||
* `no_releases` (endpoint answered but no usable version).
|
||||
* Fetch the latest GitHub release. Exported (v0.42) so the self-upgrade refresh
|
||||
* path and tests can reuse it. 5s timeout (was 10s) — this runs on the detached
|
||||
* refresh, never the hot path, but a tight bound keeps the refresh cheap.
|
||||
*/
|
||||
export async function fetchLatestRelease(): Promise<LatestReleaseResult> {
|
||||
let res: Response;
|
||||
export async function fetchLatestRelease(): Promise<{ tag: string; published_at: string; url: string } | null> {
|
||||
try {
|
||||
res = await fetch(VERSION_SOURCE_URL, {
|
||||
const res = await fetch('https://api.github.com/repos/garrytan/gbrain/releases/latest', {
|
||||
headers: { 'User-Agent': `gbrain/${VERSION}` },
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as any;
|
||||
return {
|
||||
tag: data.tag_name || '',
|
||||
published_at: data.published_at || '',
|
||||
url: data.html_url || '',
|
||||
};
|
||||
} catch {
|
||||
return { ok: false, reason: 'network_error' };
|
||||
}
|
||||
try {
|
||||
if (!res.ok) return { ok: false, reason: 'no_releases' };
|
||||
const tag = parseVersionFileBody(await res.text());
|
||||
if (!tag) return { ok: false, reason: 'no_releases' };
|
||||
return { ok: true, tag, published_at: '', url: RELEASE_NOTES_URL };
|
||||
} catch {
|
||||
return { ok: false, reason: 'network_error' };
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,33 +118,17 @@ export function extractChangelogBetween(changelog: string, from: string, to: str
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed check must NEVER write `up_to_date` — that was #486: the fetch
|
||||
* failed permanently (dead releases API) and every user was told "you're
|
||||
* current" forever. Instead, re-write the last-known-good marker (bumping its
|
||||
* mtime so the cache TTL still throttles retries and a network blip can't
|
||||
* erase a pending upgrade_available notice). No prior marker → write nothing;
|
||||
* the next invocation retries.
|
||||
*/
|
||||
function preserveCacheOnFailedCheck(): void {
|
||||
try {
|
||||
const prior = readUpdateCache();
|
||||
if (prior) safeWriteCache(prior.marker);
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the latest version and write the self-upgrade cache (the marker line
|
||||
* read by the CLI startup hook). On fetch failure the last-known-good marker is
|
||||
* preserved (see preserveCacheOnFailedCheck) — never a fabricated `up_to_date`.
|
||||
* This is the function the detached single-flight refresh (`gbrain
|
||||
* check-update --refresh-cache`) invokes.
|
||||
* Fetch the latest release and write the self-upgrade cache (the marker line
|
||||
* read by the CLI startup hook). Fail-open: on any network failure we cache
|
||||
* `UP_TO_DATE <current>` so the TTL prevents hammering GitHub on every
|
||||
* invocation. Returns the resolved marker for callers that want it. This is the
|
||||
* function the detached single-flight refresh (`gbrain check-update
|
||||
* --refresh-cache`) invokes.
|
||||
*/
|
||||
export async function refreshUpdateCache(): Promise<void> {
|
||||
const release = await fetchLatestRelease();
|
||||
if (!release.ok) {
|
||||
preserveCacheOnFailedCheck();
|
||||
if (!release) {
|
||||
safeWriteCache({ kind: 'up_to_date', current: VERSION });
|
||||
return;
|
||||
}
|
||||
const latestVersion = release.tag.replace(/^v/, '');
|
||||
@@ -209,8 +166,9 @@ export async function runCheckUpdate(args: string[]) {
|
||||
|
||||
const release = await fetchLatestRelease();
|
||||
|
||||
if (!release.ok) {
|
||||
preserveCacheOnFailedCheck();
|
||||
if (!release) {
|
||||
// Warm the cache fail-open so the startup hook doesn't re-fetch every call.
|
||||
safeWriteCache({ kind: 'up_to_date', current: VERSION });
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
current_version: VERSION,
|
||||
@@ -221,12 +179,10 @@ export async function runCheckUpdate(args: string[]) {
|
||||
release_url: '',
|
||||
changelog_diff: '',
|
||||
published_at: '',
|
||||
error: release.reason,
|
||||
error: 'no_releases',
|
||||
}, null, 2));
|
||||
} else if (release.reason === 'network_error') {
|
||||
console.log(`GBrain ${VERSION} — could not check for updates (network unavailable).`);
|
||||
} else {
|
||||
console.log(`GBrain ${VERSION} — could not determine the latest published version.`);
|
||||
console.log(`GBrain ${VERSION} — could not check for updates (no releases found or network unavailable).`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -46,14 +46,7 @@ export async function runConfig(engine: BrainEngine, args: string[]) {
|
||||
}
|
||||
console.log('GBrain config:');
|
||||
for (const [k, v] of Object.entries(config)) {
|
||||
// #575: objects interpolated into the template literal printed
|
||||
// `[object Object]` — render them as JSON instead. Sensitive keys
|
||||
// stay redacted whether the value is a string or an object.
|
||||
const display = typeof v === 'string'
|
||||
? redactConfigValue(k, v)
|
||||
: v !== null && typeof v === 'object'
|
||||
? (isSensitiveConfigKey(k) ? '***' : JSON.stringify(v))
|
||||
: v;
|
||||
const display = typeof v === 'string' ? redactConfigValue(k, v) : v;
|
||||
console.log(` ${k}: ${display}`);
|
||||
}
|
||||
return;
|
||||
|
||||
+10
-129
@@ -40,7 +40,7 @@ import {
|
||||
buildBasenameIndex,
|
||||
queryBasenameIndex,
|
||||
} from '../core/link-extraction.ts';
|
||||
import { probeSourceGitState } from '../core/git-head.ts';
|
||||
import { isSourceUnchangedSinceSync } from '../core/git-head.ts';
|
||||
// v0.41.32.0: remote staleness reads the stored newest_content_at column via
|
||||
// this pure comparator (no git subprocess on the HTTP MCP doctor path).
|
||||
import { lagFromContentMs } from '../core/source-health.ts';
|
||||
@@ -53,7 +53,6 @@ import { isUndefinedColumnError } from '../core/utils.ts';
|
||||
// drift from what search actually filters.
|
||||
import { resolveHardExcludes, DEFAULT_HARD_EXCLUDES } from '../core/search/source-boost.ts';
|
||||
import { escapeLikePattern, buildVisibilityClause } from '../core/search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from '../core/extraction-review.ts';
|
||||
import { hnswIndexExpected, hnswMaxDimsForType } from '../core/vector-index.ts';
|
||||
|
||||
export interface Check {
|
||||
@@ -3621,52 +3620,6 @@ export async function checkLinksExtractionLag(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #160 — unverified_extractions doctor check.
|
||||
*
|
||||
* The extraction quarantine lane parks auto-extracted entity stubs
|
||||
* (frontmatter `provenance: 'auto-extracted'` + `status: 'unverified'`)
|
||||
* until the owner promotes or rejects them. A queue nobody reviews decays
|
||||
* into invisible clutter, so this check counts stubs older than N days
|
||||
* (default 7) and nudges toward the review surface. Exported for direct
|
||||
* testing (mirrors checkLinksExtractionLag).
|
||||
*/
|
||||
export async function checkUnverifiedExtractions(
|
||||
engine: BrainEngine,
|
||||
opts?: { sourceId?: string; days?: number },
|
||||
): Promise<Check> {
|
||||
const name = 'unverified_extractions';
|
||||
const days = opts?.days ?? 7;
|
||||
const sourceId = opts?.sourceId;
|
||||
try {
|
||||
const params: unknown[] = [String(days)];
|
||||
let srcClause = '';
|
||||
if (sourceId) {
|
||||
params.push(sourceId);
|
||||
srcClause = 'AND p.source_id = $2';
|
||||
}
|
||||
const rows = await engine.executeRaw<{ n: string | number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND ${unverifiedExtractionFragment('p')}
|
||||
AND p.created_at < now() - ($1 || ' days')::interval
|
||||
${srcClause}`,
|
||||
params,
|
||||
);
|
||||
const n = Number(rows[0]?.n ?? 0);
|
||||
return {
|
||||
name,
|
||||
status: n > 0 ? 'warn' : 'ok',
|
||||
message: n > 0
|
||||
? `${n} unverified auto-extracted entity stub(s) older than ${days} days awaiting review. List with 'gbrain extraction-pending'; promote/reject with 'gbrain extraction-review <promote|reject> --slugs <slug,...>'.`
|
||||
: 'No stale unverified extraction stubs',
|
||||
details: { count: n, days, source_id: sourceId ?? null },
|
||||
};
|
||||
} catch (e) {
|
||||
return { name, status: 'warn', message: `Could not check unverified_extractions: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1678 — extract_atoms_backlog doctor check.
|
||||
*
|
||||
@@ -4062,51 +4015,29 @@ export async function checkSyncFreshness(
|
||||
// All four must hold; otherwise fall through to the time-based check.
|
||||
// The chunker version match is computed here (not in the helper)
|
||||
// because it depends on engine state, not git state.
|
||||
//
|
||||
// Clone-unavailable fallback: on stateless deploys (Docker on EB /
|
||||
// K8s / Fly — the platforms the cloud recipes produce), a container
|
||||
// restart wipes `local_path` and each clone is only re-materialized
|
||||
// when that source's next sync job runs. Until then the HEAD probe
|
||||
// cannot run at all ('unavailable'), which previously fell through to
|
||||
// raw wall-clock age — and since a no-op sync doesn't advance
|
||||
// `last_sync_at`, every QUIET source read as stale/FAIL after a
|
||||
// restart (score-sinking alert storm; observed live: 16-source brain,
|
||||
// 12 clones gone after a config-update restart, doctor 70→30).
|
||||
// 'unavailable' + chunker match now reuses the v0.41.32.0 REMOTE lag
|
||||
// signal (newest_content_at) below — DB-only, no subprocess, and it
|
||||
// still reports staleness whenever content really is newer than the
|
||||
// last sync. 'changed' (readable clone with real work) keeps
|
||||
// wall-clock exactly as before, and a chunker mismatch is never
|
||||
// masked (D7): it disables the fallback too.
|
||||
let cloneUnavailable = false;
|
||||
if (localOnly) {
|
||||
const gitState = probeSourceGitState(
|
||||
const gitUnchanged = isSourceUnchangedSinceSync(
|
||||
source.local_path,
|
||||
source.last_commit,
|
||||
{ requireCleanWorkingTree: 'ignore-untracked' },
|
||||
);
|
||||
const chunkerMatch = source.chunker_version === currentChunkerVersion;
|
||||
if (gitState === 'unchanged' && chunkerMatch) {
|
||||
if (gitUnchanged && chunkerMatch) {
|
||||
unchanged_count++;
|
||||
continue;
|
||||
}
|
||||
cloneUnavailable = gitState === 'unavailable' && chunkerMatch;
|
||||
}
|
||||
|
||||
// v0.41.32.0: REMOTE path (doctorReportRemote, !localOnly) computes lag
|
||||
// from the stored newest_content_at column — NO git subprocess on a
|
||||
// DB-supplied local_path (preserves the v0.41.27.0 trust boundary). A
|
||||
// quiet repo whose newest commit predates its last sync reports 0; NULL
|
||||
// column → wall-clock fallback. LOCAL fall-through keeps wall-clock when
|
||||
// the clone is READABLE: the short-circuit failed on real evidence
|
||||
// (HEAD moved / dirty tree), so the source genuinely has work and
|
||||
// "hours since last sync" is the right staleness measure. A local clone
|
||||
// that is UNAVAILABLE (not yet re-materialized, see above) carries no
|
||||
// evidence either way, so it borrows this same DB-only lag. The
|
||||
// `ageMs < 0` skew check above still runs on raw wall-clock for both
|
||||
// paths (A1).
|
||||
// column → wall-clock fallback. LOCAL fall-through keeps wall-clock: the
|
||||
// short-circuit already failed, so the source genuinely has work and
|
||||
// "hours since last sync" is the right staleness measure. The `ageMs < 0`
|
||||
// skew check above still runs on raw wall-clock for both paths (A1).
|
||||
let thresholdAgeMs = ageMs;
|
||||
if (!localOnly || cloneUnavailable) {
|
||||
if (!localOnly) {
|
||||
const contentMs = source.newest_content_at
|
||||
? new Date(source.newest_content_at).getTime()
|
||||
: null;
|
||||
@@ -4349,18 +4280,8 @@ export async function checkCycleFreshness(
|
||||
: `'${source.id}'`;
|
||||
const raw = source.config?.last_full_cycle_at;
|
||||
if (typeof raw !== 'string') {
|
||||
// #2540: WARN, not FAIL. This check iterates EVERY local_path source,
|
||||
// so on a multi-source install where only some vaults are cycled
|
||||
// (e.g. one nightly `gbrain dream --dir <vault>`), a never-cycled
|
||||
// sibling source turned doctor permanently red — which erodes the
|
||||
// check's signal until real staleness hides inside the noise (the
|
||||
// reporter's install masked genuinely stale sources for weeks this
|
||||
// way). "Never cycled" also fires on a source added minutes ago.
|
||||
// A source that HAS cycled and then went stale still escalates
|
||||
// through the warn/fail age thresholds below — that is the
|
||||
// regression signal this check exists for.
|
||||
issues.push(`Source ${display} has never completed a full cycle`);
|
||||
hasWarnings = true;
|
||||
hasFailures = true;
|
||||
continue;
|
||||
}
|
||||
const last = new Date(raw).getTime();
|
||||
@@ -4396,7 +4317,7 @@ export async function checkCycleFreshness(
|
||||
return {
|
||||
name: 'cycle_freshness',
|
||||
status: 'warn',
|
||||
message: `${issues.join('; ')}. Run \`gbrain dream --source <id>\` to cycle a source, or start \`gbrain autopilot\`.`,
|
||||
message: `${issues.join('; ')}.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
@@ -5602,42 +5523,6 @@ export async function buildChecks(
|
||||
// Best-effort filesystem-hygiene check; never block doctor.
|
||||
}
|
||||
|
||||
// 3f. npm_squat (#505). The npm registry name `gbrain` belongs to an
|
||||
// unrelated third-party package — this project is NOT distributed on npm.
|
||||
// A reflexive `npm i -g gbrain` / `bun add -g gbrain` installs something
|
||||
// unrelated that can shadow the real binary on PATH. Classify every
|
||||
// `gbrain` that `which -a` finds (pure helpers in
|
||||
// src/core/npm-squat-check.ts) and warn when an unrelated install wins on
|
||||
// PATH or the entry is broken. Skips silently when gbrain isn't on PATH
|
||||
// at all (e.g. running via `bun src/cli.ts`).
|
||||
try {
|
||||
const { execSync } = await import('node:child_process');
|
||||
let candidates: string[] = [];
|
||||
try {
|
||||
candidates = execSync('which -a gbrain', {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
})
|
||||
.split('\n')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
} catch {
|
||||
// `which` exits non-zero when gbrain isn't on PATH (or is missing
|
||||
// entirely on this platform) — nothing to check.
|
||||
}
|
||||
const { assessGbrainBinaries } = await import('../core/npm-squat-check.ts');
|
||||
const assessment = assessGbrainBinaries(candidates);
|
||||
if (assessment.status !== 'skip') {
|
||||
checks.push({
|
||||
name: 'npm_squat',
|
||||
status: assessment.status,
|
||||
message: assessment.message,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort environment check; never block doctor.
|
||||
}
|
||||
|
||||
// 3b-multi-source. Multi-source drift (v0.31.8 — D8 + D17 + OV12 + OV13).
|
||||
// Pre-v0.30.3 putPage misrouted multi-source writes to (default, slug).
|
||||
// For each non-default source with local_path set, walk the FS and surface
|
||||
@@ -7006,10 +6891,6 @@ export async function buildChecks(
|
||||
checks.push({ name: 'flagged_pages', status: 'ok', message: `Skipped (${msg})` });
|
||||
}
|
||||
|
||||
// issue #160: extraction quarantine lane review nudge.
|
||||
progress.heartbeat('unverified_extractions');
|
||||
checks.push(await checkUnverifiedExtractions(engine, { sourceId: orphanRatioSourceId }));
|
||||
|
||||
// 11a. Frontmatter integrity (v0.22.4, hardened in v0.38.2.0).
|
||||
// scanBrainSources walks every registered source's local_path on disk
|
||||
// (not from the DB), invoking parseMarkdown(..., {validate:true}) per
|
||||
|
||||
+3
-50
@@ -19,26 +19,6 @@ import {
|
||||
} from '../core/pace-mode.ts';
|
||||
import { tryAcquireDbLock, type DbLockHandle } from '../core/db-lock.ts';
|
||||
import { embedBackfillLockId } from '../core/embed-backfill-lock.ts';
|
||||
import { wrapChunkTextsForStoredMode } from '../core/embedding-context.ts';
|
||||
import { titleTierCorpusGeneration } from '../core/contextual-retrieval-service.ts';
|
||||
import type { Page } from '../core/types.ts';
|
||||
|
||||
/**
|
||||
* #3507 — after a plain re-embed fully re-embedded a `per_chunk_synopsis`
|
||||
* page at the title-only tier (see wrapChunkTextsForStoredMode), restamp the
|
||||
* page's CR state to 'title' so `contextual_retrieval_mode` keeps describing
|
||||
* the vectors actually in the column. The reindex sweep restores the synopsis
|
||||
* tier later. No-op for every other mode.
|
||||
*/
|
||||
export async function restampIfDemotedToTitleTier(
|
||||
engine: BrainEngine,
|
||||
page: Pick<Page, 'contextual_retrieval_mode'> | null | undefined,
|
||||
slug: string,
|
||||
sourceId: string,
|
||||
): Promise<void> {
|
||||
if (page?.contextual_retrieval_mode !== 'per_chunk_synopsis') return;
|
||||
await engine.updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration());
|
||||
}
|
||||
|
||||
export interface EmbedOpts {
|
||||
/** Embed ALL pages (every chunk). */
|
||||
@@ -619,11 +599,7 @@ async function embedPage(
|
||||
return;
|
||||
}
|
||||
|
||||
// #3507: embed with the page's STORED wrapping convention (title-tier
|
||||
// contextual prefix when the page was embedded wrapped), not raw
|
||||
// chunk_text — otherwise a re-embed silently strips the contextual
|
||||
// prefixes the sync path applied. fenced_code chunks stay unwrapped.
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed), { abortSignal: signal });
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text), { abortSignal: signal });
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
@@ -646,9 +622,6 @@ async function embedPage(
|
||||
// such a page and then stamps it.
|
||||
if (toEmbed.length === chunks.length) {
|
||||
await engine.setPageEmbeddingSignature(slug, { sourceId, signature: currentEmbeddingSignature() });
|
||||
// #3507: a fully re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest.
|
||||
await restampIfDemotedToTitleTier(engine, page, slug, page.source_id);
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
@@ -790,8 +763,7 @@ async function embedAll(
|
||||
}
|
||||
|
||||
try {
|
||||
// #3507: reproduce the page's stored wrapping convention (see embedPage).
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed));
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text));
|
||||
// Build a map of new embeddings by chunk_index
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
@@ -813,11 +785,6 @@ async function embedAll(
|
||||
await observed(pacer, () =>
|
||||
engine.setPageEmbeddingSignature(page.slug, { sourceId: pageSourceId, signature }),
|
||||
);
|
||||
// #3507: --all fully re-embeds; a per_chunk_synopsis page landed at
|
||||
// the title tier — keep the stamped mode honest.
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, page, page.slug, pageSourceId),
|
||||
);
|
||||
result.embedded += toEmbed.length;
|
||||
} catch (e: unknown) {
|
||||
serr(`\n Error embedding ${page.slug}: ${e instanceof Error ? e.message : e}`);
|
||||
@@ -1131,13 +1098,7 @@ async function embedAllStale(
|
||||
const keySourceId = stale[0]?.source_id ?? 'default';
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes — `embed --stale` is the
|
||||
// NORMAL post-model-migration path, so raw-text embedding here
|
||||
// quietly converted whole corpora to the unwrapped convention.
|
||||
const pageRow = await observed(pacer, () => engine.getPage(slug, { sourceId: keySourceId }));
|
||||
const embeddings = await embedBatchWithBackoff(wrapChunkTextsForStoredMode(pageRow, stale), { abortSignal: effectiveSignal });
|
||||
const embeddings = await embedBatchWithBackoff(stale.map(c => c.chunk_text), { abortSignal: effectiveSignal });
|
||||
// Re-fetch existing chunks and merge to avoid deleting non-stale chunks.
|
||||
const existing = await observed(pacer, () => engine.getChunks(slug, { sourceId: keySourceId }));
|
||||
const staleIdxToEmbedding = new Map<number, Float32Array>();
|
||||
@@ -1165,14 +1126,6 @@ async function embedAllStale(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest. Partially-stale pages
|
||||
// stay stamped as-is (mixed provenance; reindex sweeps fix them).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
} catch (e: unknown) {
|
||||
// Budget/abort-fired cancellations are expected on the way out; don't
|
||||
|
||||
@@ -78,7 +78,7 @@ FLAGS:
|
||||
cycle is 3 model calls; verdict aggregates over them.
|
||||
--slot-a-model <id> Override default 'openai:gpt-5.2'.
|
||||
--slot-b-model <id> Override default 'anthropic:claude-opus-4-7'.
|
||||
--slot-c-model <id> Override default 'deepseek:deepseek-v4-pro'.
|
||||
--slot-c-model <id> Override default 'google:gemini-1.5-pro'.
|
||||
--receipt-dir <path> Default: gbrainPath('eval-receipts').
|
||||
--max-tokens N Output token budget per call. Default: 4000.
|
||||
--json Emit final aggregate as JSON to stdout (progress to stderr).
|
||||
|
||||
@@ -149,40 +149,6 @@ export const ALLOWED_TYPES = [
|
||||
] as const;
|
||||
export type AllowedType = (typeof ALLOWED_TYPES)[number];
|
||||
|
||||
/**
|
||||
* Granular collector page-types that alias into each canonical conversation
|
||||
* bucket. The v2 type-consolidation pack retypes these to the canonical names
|
||||
* (`slack-dm-day`/`slack-thread` → `slack`, `email-digest` → `email`), but a
|
||||
* brain that hasn't run that pack still carries the collector's granular types
|
||||
* in `pages.type`. Without this expansion, `listPages({ type: 'slack' })`
|
||||
* matches zero rows on such brains and the whole comms corpus is silently
|
||||
* skipped (facts stay empty → `find_trajectory` returns nothing). The canonical
|
||||
* name is always included first so consolidated brains keep working unchanged.
|
||||
*/
|
||||
export const ALLOWED_TYPE_ALIASES: Record<AllowedType, readonly string[]> = {
|
||||
conversation: ['conversation'],
|
||||
meeting: ['meeting'],
|
||||
slack: ['slack', 'slack-dm-day', 'slack-thread'],
|
||||
email: ['email', 'email-digest'],
|
||||
imessage: ['imessage'],
|
||||
'imessage-daily': ['imessage-daily'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Expand the requested logical types to the concrete `pages.type` values to
|
||||
* enumerate, canonical-first and de-duplicated. Unknown types pass through
|
||||
* unchanged so an explicit override is never dropped.
|
||||
*/
|
||||
export function pageTypesForAllowed(types: readonly AllowedType[]): string[] {
|
||||
const out: string[] = [];
|
||||
for (const t of types) {
|
||||
for (const concrete of ALLOWED_TYPE_ALIASES[t] ?? [t]) {
|
||||
if (!out.includes(concrete)) out.push(concrete);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pagination batch size for listPages enumeration. Per-batch memory
|
||||
* worst case = BATCH × MAX_PAGE_BODY_BYTES = 250MB at default 10
|
||||
@@ -1298,18 +1264,13 @@ export async function runExtractConversationFactsCore(
|
||||
}
|
||||
};
|
||||
|
||||
// Expand logical types (conversation/meeting/slack/email) to the concrete
|
||||
// `pages.type` values to enumerate, so brains on the granular collector
|
||||
// types are not silently skipped (see ALLOWED_TYPE_ALIASES).
|
||||
const concreteTypes = pageTypesForAllowed(types);
|
||||
|
||||
if (opts.slug) {
|
||||
const page = await engine.getPage(opts.slug, { sourceId });
|
||||
if (!page) {
|
||||
result.pages_skipped_disappeared++;
|
||||
return;
|
||||
}
|
||||
if (!concreteTypes.includes(page.type)) {
|
||||
if (!types.includes(page.type as AllowedType)) {
|
||||
result.pages_skipped++;
|
||||
return;
|
||||
}
|
||||
@@ -1323,7 +1284,7 @@ export async function runExtractConversationFactsCore(
|
||||
// honors AbortSignal at each claim boundary and threads
|
||||
// BudgetExhausted abort (D13) automatically.
|
||||
let processedPagesCount = 0;
|
||||
pageLoop: for (const type of concreteTypes) {
|
||||
pageLoop: for (const type of types) {
|
||||
let offset = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
|
||||
+5
-30
@@ -35,7 +35,7 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput } from '../core/en
|
||||
import type { PageType } from '../core/types.ts';
|
||||
import { parseMarkdown } from '../core/markdown.ts';
|
||||
import {
|
||||
extractPageLinks, parseTimelineEntries, deriveTimelineAnchor, inferLinkType, makeResolver,
|
||||
extractPageLinks, parseTimelineEntries, inferLinkType, makeResolver,
|
||||
extractFrontmatterLinks, isGlobalBasenameEnabled, LINK_EXTRACTOR_VERSION_TS,
|
||||
WIKILINK_BASENAME_LINK_TYPE,
|
||||
buildBasenameIndex, queryBasenameIndex, stripCodeBlocks,
|
||||
@@ -349,13 +349,7 @@ function inferTypeByDir(fromDir: string, toDir: string, frontmatter?: Record<str
|
||||
const to = toDir.split('/')[0];
|
||||
if (from === 'people' && to === 'companies') {
|
||||
if (Array.isArray(frontmatter?.founded)) return 'founded';
|
||||
// #3466: bare people/ -> companies/ adjacency is not evidence of
|
||||
// employment, so it gets the neutral 'mentions' verb instead of
|
||||
// 'works_at'. Real works_at edges still come from the two paths that
|
||||
// read actual evidence: the company:/companies: frontmatter fields
|
||||
// (FRONTMATTER_LINK_MAP) and employment phrasing in prose
|
||||
// (inferLinkType in link-extraction.ts).
|
||||
return 'mentions';
|
||||
return 'works_at';
|
||||
}
|
||||
if (from === 'people' && to === 'deals') return 'involved_in';
|
||||
if (from === 'deals' && to === 'companies') return 'deal_for';
|
||||
@@ -755,12 +749,6 @@ export async function runExtract(engine: BrainEngine, args: string[]) {
|
||||
// v0.41.18.0 (A11, T8): --from-meetings extracts timeline entries from
|
||||
// meeting pages onto each discussed entity. Timeline subcommand only.
|
||||
const fromMeetings = args.includes('--from-meetings');
|
||||
// --infer-dates: for pages whose body has NO parseable timeline line, anchor
|
||||
// one entry at the page's computed effective_date (frontmatter / filename date,
|
||||
// never the updated_at fallback). Default OFF for back-compat — comms/calendar
|
||||
// brains opt in to populate timeline from slug/frontmatter dates. DB-source only
|
||||
// (needs the full Page.effective_date, which getPage projects).
|
||||
const inferDates = args.includes('--infer-dates');
|
||||
// v0.41.17.0 (T7, D9): --workers N parsed via the shared validator.
|
||||
// Honored on the fs-walk inner loops only; DB-source paths stay
|
||||
// serial in v0.41.17.0 (see ExtractOpts.workers doc).
|
||||
@@ -975,7 +963,7 @@ Status (v0.42):
|
||||
result.pages_processed = r.pages;
|
||||
}
|
||||
if (subcommand === 'timeline' || subcommand === 'all') {
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter, inferDates });
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter });
|
||||
result.timeline_entries_created = r.created;
|
||||
result.pages_processed = Math.max(result.pages_processed, r.pages);
|
||||
}
|
||||
@@ -1595,7 +1583,7 @@ async function extractTimelineFromDB(
|
||||
jsonMode: boolean,
|
||||
typeFilter: PageType | undefined,
|
||||
since: string | undefined,
|
||||
opts?: { sourceIdFilter?: string; inferDates?: boolean },
|
||||
opts?: { sourceIdFilter?: string },
|
||||
): Promise<{ created: number; pages: number }> {
|
||||
// v0.32.8: listAllPageRefs enumerates (slug, source_id) pairs so we can
|
||||
// thread sourceId to getPage and addTimelineEntriesBatch. Pre-fix used
|
||||
@@ -1604,7 +1592,6 @@ async function extractTimelineFromDB(
|
||||
// v0.37.7.0 #1204: when sourceIdFilter is set, scope the walk to one
|
||||
// source so federated brain users can extract per-source.
|
||||
const sourceIdFilter = opts?.sourceIdFilter;
|
||||
const inferDates = opts?.inferDates ?? false;
|
||||
const allRefs = sourceIdFilter
|
||||
? (await engine.listAllPageRefs()).filter(r => r.source_id === sourceIdFilter)
|
||||
: await engine.listAllPageRefs();
|
||||
@@ -1644,19 +1631,7 @@ async function extractTimelineFromDB(
|
||||
}
|
||||
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
let entries = parseTimelineEntries(fullContent);
|
||||
// --infer-dates: pages with no in-body timeline line but a trustworthy
|
||||
// content date (frontmatter / filename) get one anchor entry at that date.
|
||||
// Applied ONLY on the zero-entry path so it never shadows a real timeline.
|
||||
if (entries.length === 0 && inferDates) {
|
||||
const anchor = deriveTimelineAnchor({
|
||||
slug,
|
||||
title: page.title,
|
||||
effectiveDate: page.effective_date,
|
||||
effectiveDateSource: page.effective_date_source,
|
||||
});
|
||||
if (anchor) entries = [anchor];
|
||||
}
|
||||
const entries = parseTimelineEntries(fullContent);
|
||||
|
||||
for (const entry of entries) {
|
||||
if (dryRunSeen) {
|
||||
|
||||
@@ -42,7 +42,7 @@ interface FeatureScanResult {
|
||||
const RECIPE_META = [
|
||||
{ id: 'email-to-brain', name: 'Email to Brain', secrets: ['GMAIL_APP_PASSWORD'] },
|
||||
{ id: 'calendar-to-brain', name: 'Calendar Sync', secrets: ['GOOGLE_CALENDAR_API_KEY'] },
|
||||
{ id: 'x-to-brain', name: 'X/Twitter to Brain', secrets: ['X_API_BEARER_TOKEN'] },
|
||||
{ id: 'x-to-brain', name: 'X/Twitter to Brain', secrets: ['X_BEARER_TOKEN'] },
|
||||
{ id: 'twilio-voice-brain', name: 'Voice to Brain', secrets: ['TWILIO_AUTH_TOKEN'] },
|
||||
{ id: 'meeting-sync', name: 'Meeting Sync', secrets: ['CIRCLEBACK_API_KEY'] },
|
||||
{ id: 'credential-gateway', name: 'Credential Gateway', secrets: ['OAUTH_CLIENT_SECRET'] },
|
||||
|
||||
+2
-10
@@ -16,7 +16,7 @@ interface FileRecord {
|
||||
filename: string;
|
||||
storage_path: string;
|
||||
mime_type: string | null;
|
||||
size_bytes: number | bigint | string | null;
|
||||
size_bytes: number;
|
||||
content_hash: string;
|
||||
metadata: Record<string, unknown>;
|
||||
created_at: string;
|
||||
@@ -42,14 +42,6 @@ function fileHash(filePath: string): string {
|
||||
return createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
export function formatFileSizeKb(rawSizeBytes: number | bigint | string | null): string {
|
||||
if (rawSizeBytes == null) return '?';
|
||||
const sizeBytes = Number(rawSizeBytes);
|
||||
return Number.isFinite(sizeBytes) && sizeBytes >= 0
|
||||
? `${Math.round(sizeBytes / 1024)}KB`
|
||||
: '?';
|
||||
}
|
||||
|
||||
export async function runFiles(engine: BrainEngine, args: string[]) {
|
||||
const subcommand = args[0];
|
||||
|
||||
@@ -124,7 +116,7 @@ async function listFiles(engine: BrainEngine, slug?: string) {
|
||||
|
||||
console.log(`${rows.length} file(s):`);
|
||||
for (const row of rows) {
|
||||
const size = formatFileSizeKb(row.size_bytes as FileRecord['size_bytes']);
|
||||
const size = row.size_bytes ? `${Math.round(Number(row.size_bytes) / 1024)}KB` : '?';
|
||||
console.log(` ${row.page_slug || '(unlinked)'} / ${row.filename} [${size}, ${row.mime_type || '?'}]`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,8 +23,7 @@ import matter from 'gray-matter';
|
||||
import { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync } from 'fs';
|
||||
import { join, basename } from 'path';
|
||||
import { homedir } from 'os';
|
||||
import { gbrainPath, loadConfig } from '../core/config.ts';
|
||||
import { buildGatewayConfig } from '../core/ai/build-gateway-config.ts';
|
||||
import { gbrainPath } from '../core/config.ts';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
// --- Types ---
|
||||
@@ -123,28 +122,9 @@ export function isUnsafeHealthCheck(check: string): boolean {
|
||||
return /[;&|`$(){}\\<>\n]/.test(check);
|
||||
}
|
||||
|
||||
/**
|
||||
* Env view for secret resolution (#2789): apply the same config.json→env
|
||||
* folding the runtime applies via buildGatewayConfig, so a credential stored
|
||||
* only in ~/.gbrain/config.json — which powers a perfectly healthy
|
||||
* integration — is not reported [missing] by show/status. process.env still
|
||||
* wins for non-empty values (buildGatewayConfig spreads it last, dropping
|
||||
* only ''/undefined entries). Falls back to bare process.env before
|
||||
* `gbrain init` (no config file yet). Mirrors the #2728 fix on the
|
||||
* providers command.
|
||||
*/
|
||||
export function secretEnv(): Record<string, string | undefined> {
|
||||
try {
|
||||
const cfg = loadConfig();
|
||||
if (cfg) return buildGatewayConfig(cfg).env;
|
||||
} catch { /* integrations must keep working pre-init — fall through */ }
|
||||
return process.env;
|
||||
}
|
||||
|
||||
/** Expand $VAR references with gateway-env (config-folded) values */
|
||||
/** Expand $VAR references with process.env values */
|
||||
export function expandVars(s: string): string {
|
||||
const env = secretEnv();
|
||||
return s.replace(/\$([A-Z_][A-Z0-9_]*)/g, (_, name) => env[name] || '');
|
||||
return s.replace(/\$([A-Z_][A-Z0-9_]*)/g, (_, name) => process.env[name] || '');
|
||||
}
|
||||
|
||||
// --- SSRF Protection ---
|
||||
@@ -269,7 +249,7 @@ export async function executeHealthCheck(
|
||||
}
|
||||
|
||||
case 'env_exists': {
|
||||
const val = secretEnv()[check.name];
|
||||
const val = process.env[check.name];
|
||||
return {
|
||||
...base,
|
||||
status: val ? 'ok' : 'fail',
|
||||
@@ -477,12 +457,11 @@ function readHeartbeat(id: string): HeartbeatEntry[] {
|
||||
|
||||
// --- Secret Checking ---
|
||||
|
||||
export function checkSecrets(secrets: RecipeSecret[]): { set: string[]; missing: RecipeSecret[] } {
|
||||
function checkSecrets(secrets: RecipeSecret[]): { set: string[]; missing: RecipeSecret[] } {
|
||||
const set: string[] = [];
|
||||
const missing: RecipeSecret[] = [];
|
||||
const env = secretEnv();
|
||||
for (const s of secrets) {
|
||||
if (env[s.name]) {
|
||||
if (process.env[s.name]) {
|
||||
set.push(s.name);
|
||||
} else {
|
||||
missing.push(s);
|
||||
@@ -628,9 +607,8 @@ function cmdShow(args: string[]): void {
|
||||
if (f.requires.length > 0) console.log(`Requires: ${f.requires.join(', ')}`);
|
||||
|
||||
console.log('\nSecrets needed:');
|
||||
const env = secretEnv();
|
||||
for (const s of f.secrets) {
|
||||
const isSet = env[s.name] ? ' [set]' : ' [missing]';
|
||||
const isSet = process.env[s.name] ? ' [set]' : ' [missing]';
|
||||
console.log(` ${s.name}${isSet}`);
|
||||
console.log(` ${s.description}`);
|
||||
console.log(` Get it: ${s.where}`);
|
||||
|
||||
+5
-37
@@ -143,31 +143,6 @@ export function resolveWorkerConcurrency(args: string[], env: NodeJS.ProcessEnv
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* #3026: the thin-client `list`/`get` branches receive jobs as parsed JSON
|
||||
* off the MCP wire, where every timestamp is an ISO string — but formatJob /
|
||||
* formatJobDetail (and the stalled-detection comparison) hold a Date
|
||||
* contract, hydrated locally by MinionQueue.rowToJob. Rehydrate once at the
|
||||
* unpack boundary so both paths hand the formatters real Dates. Exported for
|
||||
* unit tests.
|
||||
*/
|
||||
const JOB_DATE_FIELDS = [
|
||||
'created_at', 'updated_at', 'started_at', 'finished_at', 'lock_until', 'delay_until',
|
||||
] as const;
|
||||
|
||||
export function rehydrateJobDates<T>(job: T): T {
|
||||
if (!job || typeof job !== 'object') return job;
|
||||
const rec = job as { [k: string]: unknown };
|
||||
for (const field of JOB_DATE_FIELDS) {
|
||||
const v = rec[field];
|
||||
if (typeof v === 'string') {
|
||||
const d = new Date(v);
|
||||
if (!Number.isNaN(d.getTime())) rec[field] = d;
|
||||
}
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
function formatJob(job: MinionJob): string {
|
||||
const dur = job.finished_at && job.started_at
|
||||
? `${((job.finished_at.getTime() - job.started_at.getTime()) / 1000).toFixed(1)}s`
|
||||
@@ -233,7 +208,7 @@ USAGE
|
||||
gbrain jobs get <id>
|
||||
gbrain jobs cancel <id>
|
||||
gbrain jobs retry <id>
|
||||
gbrain jobs prune [--older-than 30d] [--dry-run]
|
||||
gbrain jobs prune [--older-than 30d]
|
||||
gbrain jobs delete <id>
|
||||
gbrain jobs stats
|
||||
gbrain jobs smoke
|
||||
@@ -521,7 +496,7 @@ HANDLER TYPES (built in)
|
||||
const raw = await callRemoteTool(cfg!, 'list_jobs', {
|
||||
status, queue: queueName, limit,
|
||||
}, { timeoutMs: 30_000 });
|
||||
jobs = unpackToolResult<MinionJob[]>(raw).map((j) => rehydrateJobDates(j));
|
||||
jobs = unpackToolResult<MinionJob[]>(raw);
|
||||
} else {
|
||||
try { await queue.ensureSchema(); }
|
||||
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
|
||||
@@ -550,7 +525,7 @@ HANDLER TYPES (built in)
|
||||
if (isThinClient(cfg)) {
|
||||
try {
|
||||
const raw = await callRemoteTool(cfg!, 'get_job', { id }, { timeoutMs: 30_000 });
|
||||
job = rehydrateJobDates(unpackToolResult<MinionJob | null>(raw));
|
||||
job = unpackToolResult<MinionJob | null>(raw);
|
||||
} catch (e) {
|
||||
// The remote op throws `invalid_params` on not-found; surface as
|
||||
// the same "Job not found" exit-1 the local path produces.
|
||||
@@ -633,15 +608,8 @@ HANDLER TYPES (built in)
|
||||
try { await queue.ensureSchema(); }
|
||||
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
|
||||
|
||||
// #2712: --dry-run previews the count without deleting. It used to be
|
||||
// silently ignored (the destructive default ran anyway).
|
||||
const dryRun = hasFlag(args, '--dry-run');
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000), dryRun });
|
||||
if (dryRun) {
|
||||
console.log(`[dry-run] Would prune ${count} jobs older than ${days} days. Nothing deleted.`);
|
||||
} else {
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
}
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000) });
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -151,17 +151,8 @@ export async function runReindexFrontmatter(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI entrypoint. Argv shape matches reindex-code for consistency.
|
||||
*
|
||||
* #1963: takes the ALREADY-CONNECTED engine from cli.ts's dispatch instead of
|
||||
* building its own. The old self-managed `createEngine()+connect()` here was a
|
||||
* same-process double-connect: cli.ts's `connectEngine()` already held the
|
||||
* PGLite data-dir lock, so the second `connect()` spun the full 30s lock
|
||||
* timeout waiting on its own process and the command always exited 1 on
|
||||
* PGLite. The engine lifecycle (connect + teardown) belongs to cli.ts.
|
||||
*/
|
||||
export async function reindexFrontmatterCli(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
/** CLI entrypoint. Argv shape matches reindex-code for consistency. */
|
||||
export async function reindexFrontmatterCli(args: string[]): Promise<void> {
|
||||
const opts: ReindexFrontmatterOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
@@ -182,15 +173,37 @@ export async function reindexFrontmatterCli(engine: BrainEngine, args: string[])
|
||||
}
|
||||
}
|
||||
|
||||
const result = await runReindexFrontmatter(engine, opts);
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
const noun = result.status === 'dry_run' ? 'would update' : 'updated';
|
||||
console.error(
|
||||
`\nReindex ${result.status}: examined=${result.examined} ${noun}=${result.updated} ` +
|
||||
`fallback=${result.fallback} dur=${result.durationSec.toFixed(1)}s`,
|
||||
);
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const { loadConfig, toEngineConfig } = await import('../core/config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) {
|
||||
console.error('No gbrain config; run `gbrain init` first.');
|
||||
process.exit(1);
|
||||
}
|
||||
const engineConfig = toEngineConfig(cfg);
|
||||
const engine = await createEngine(engineConfig);
|
||||
// v0.37.7.0 #1225: createEngine() only constructs; callers MUST connect
|
||||
// before any executeRaw call. Pre-fix, the first query in countAffected
|
||||
// crashed with "PGLite not connected. Call connect() first." even on
|
||||
// --dry-run. initSchema is idempotent on a current schema, costs ~1ms.
|
||||
await engine.connect(engineConfig);
|
||||
await engine.initSchema();
|
||||
|
||||
try {
|
||||
const result = await runReindexFrontmatter(engine, opts);
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
const noun = result.status === 'dry_run' ? 'would update' : 'updated';
|
||||
console.error(
|
||||
`\nReindex ${result.status}: examined=${result.examined} ${noun}=${result.updated} ` +
|
||||
`fallback=${result.fallback} dur=${result.durationSec.toFixed(1)}s`,
|
||||
);
|
||||
}
|
||||
if (result.status === 'cancelled') process.exit(1);
|
||||
} finally {
|
||||
if ('disconnect' in engine && typeof engine.disconnect === 'function') {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}
|
||||
if (result.status === 'cancelled') process.exit(1);
|
||||
}
|
||||
|
||||
@@ -35,8 +35,7 @@ export async function runSelfUpgrade(args: string[]): Promise<void> {
|
||||
const force = args.includes('--force');
|
||||
const json = args.includes('--json');
|
||||
|
||||
const result = await fetchLatestRelease();
|
||||
const release = result.ok ? result : null;
|
||||
const release = await fetchLatestRelease();
|
||||
const latest = release ? release.tag.replace(/^v/, '') : null;
|
||||
const behind = !!latest && isValidVersionString(latest) && isNewerVersion(VERSION, latest);
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
|
||||
import express from 'express';
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import type { Server as HttpServer } from 'http';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import cors from 'cors';
|
||||
import rateLimit from 'express-rate-limit';
|
||||
@@ -47,7 +46,6 @@ import {
|
||||
type IngestionEvent,
|
||||
} from '../core/ingestion/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
|
||||
/**
|
||||
* /health endpoint timeout. 3s rather than 5s: Fly.io's default
|
||||
@@ -57,71 +55,6 @@ import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
*/
|
||||
export const HEALTH_TIMEOUT_MS = 3000;
|
||||
|
||||
/** Exported so tests can type their structural fakes exactly (#3599). */
|
||||
export type HttpServerLifecycle = Pick<HttpServer, 'listening' | 'once' | 'off' | 'close'>;
|
||||
/** Exported so tests can type their structural fakes exactly (#3599). */
|
||||
export type SignalSource = Pick<NodeJS.Process, 'once' | 'off'>;
|
||||
type CleanupRegistrar = typeof registerCleanup;
|
||||
|
||||
/**
|
||||
* Keep the HTTP server strongly referenced and make the daemon lifetime
|
||||
* explicit instead of relying on runtime-specific event-loop behavior for an
|
||||
* unobserved `app.listen()` return value. The shared abnormal-termination
|
||||
* cleanup pass closes it before process exit.
|
||||
*/
|
||||
export function waitForHttpServerLifecycle(
|
||||
server: HttpServerLifecycle,
|
||||
options: {
|
||||
signals?: SignalSource;
|
||||
register?: CleanupRegistrar;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const signals = options.signals ?? process;
|
||||
const register = options.register ?? registerCleanup;
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let closePromise: Promise<void> | null = null;
|
||||
|
||||
const closeServer = (): Promise<void> => {
|
||||
if (closePromise) return closePromise;
|
||||
closePromise = new Promise<void>((closeResolve, closeReject) => {
|
||||
if (!server.listening) {
|
||||
closeResolve();
|
||||
return;
|
||||
}
|
||||
server.close((error?: Error) => {
|
||||
if (error) closeReject(error);
|
||||
else closeResolve();
|
||||
});
|
||||
});
|
||||
return closePromise;
|
||||
};
|
||||
|
||||
const deregister = register('http-server', closeServer);
|
||||
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
server.off('close', onClose);
|
||||
server.off('error', onError);
|
||||
signals.off('SIGINT', onSigint);
|
||||
deregister();
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
};
|
||||
const onClose = () => finish();
|
||||
const onError = (error: Error) => finish(error);
|
||||
const onSigint = () => {
|
||||
void closeServer().catch(onError);
|
||||
};
|
||||
|
||||
server.once('close', onClose);
|
||||
server.once('error', onError);
|
||||
signals.once('SIGINT', onSigint);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.36.1.x #1024: bootstrap token resolution.
|
||||
*
|
||||
@@ -202,25 +135,6 @@ export type ProbeHealthResult =
|
||||
| { ok: true; status: 200; body: { status: 'ok'; version: string; engine: string; [k: string]: unknown } }
|
||||
| { ok: false; status: 503; body: { error: 'service_unavailable'; error_description: string } };
|
||||
|
||||
/** Exported so tests can type their structural fakes exactly (#3598). */
|
||||
export type AdminSseResponse = Pick<Response, 'setHeader' | 'flushHeaders' | 'write'>;
|
||||
|
||||
/**
|
||||
* Complete the admin EventSource handshake immediately.
|
||||
*
|
||||
* `flushHeaders()` alone can leave reverse proxies and browsers waiting for
|
||||
* the first response body bytes. An SSE comment is protocol-valid, ignored by
|
||||
* EventSource consumers, and makes the stream observable end-to-end without
|
||||
* fabricating an application event.
|
||||
*/
|
||||
export function openAdminSseStream(res: AdminSseResponse): void {
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.flushHeaders();
|
||||
res.write(': connected\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure async health probe. Races `engine.getStats()` against a timeout,
|
||||
* returns a tagged result. No Express coupling — easy to unit-test with a
|
||||
@@ -1718,7 +1632,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// SSE live activity feed
|
||||
// ---------------------------------------------------------------------------
|
||||
app.get('/admin/events', requireAdmin, (req: Request, res: Response) => {
|
||||
openAdminSseStream(res);
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.flushHeaders();
|
||||
|
||||
sseClients.add(res);
|
||||
req.on('close', () => sseClients.delete(res));
|
||||
@@ -2493,7 +2410,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// ---------------------------------------------------------------------------
|
||||
const clientCount = await sql`SELECT count(*)::int as count FROM oauth_clients`;
|
||||
|
||||
const httpServer = app.listen(port, bind, () => {
|
||||
app.listen(port, bind, () => {
|
||||
console.error(`
|
||||
╔══════════════════════════════════════════════════════╗
|
||||
║ GBrain MCP Server v${VERSION.padEnd(37)}║
|
||||
@@ -2518,6 +2435,4 @@ ${bootstrapFromEnv
|
||||
: `║ Admin Token (paste into /admin login): ║\n║ ${bootstrapToken.substring(0, 50)} ║\n║ ${bootstrapToken.substring(50).padEnd(50)} ║\n╚══════════════════════════════════════════════════════╝`}
|
||||
`);
|
||||
});
|
||||
|
||||
await waitForHttpServerLifecycle(httpServer);
|
||||
}
|
||||
|
||||
+4
-81
@@ -2874,17 +2874,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
: await resolveSlugByPathOrSourcePath(engine, from, undefined);
|
||||
// The new path doesn't yet have a row, so resolve from path only.
|
||||
const newSlug = resolveSlugForPath(to);
|
||||
// #3056: the cheap rename is OBSERVED, not assumed. A zero-row UPDATE
|
||||
// doesn't throw, and a thrown collision used to be swallowed by an
|
||||
// empty catch — both fell through to importFile, which created/updated
|
||||
// the row at the new path while the old row stayed behind live. Both
|
||||
// shapes now fall through to the reconcile below.
|
||||
let renameApplied = false;
|
||||
try {
|
||||
renameApplied = (await engine.updateSlug(oldSlug, newSlug, renameOpts)) > 0;
|
||||
await engine.updateSlug(oldSlug, newSlug, renameOpts);
|
||||
} catch {
|
||||
// Destination slug occupied or invalid — treat as add; the reconcile
|
||||
// below removes the stale old row once the destination materialized.
|
||||
// Slug doesn't exist or collision, treat as add
|
||||
}
|
||||
// Reimport at new path (picks up content changes). Wrapped to match the
|
||||
// deletes/adds loops: a malformed renamed file is recorded to failedFiles
|
||||
@@ -2897,11 +2890,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// NAV-1 TOCTOU: refuse a destination that realpath-resolves outside the
|
||||
// repo (committed symlink pointing out).
|
||||
const filePath = join(gitContextRoot, to);
|
||||
let importResult: Awaited<ReturnType<typeof importFile>> | undefined;
|
||||
if (existsSync(filePath) && isPathSafe(filePath, gitContextRoot)) {
|
||||
try {
|
||||
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
|
||||
importResult = result;
|
||||
if (result.status === 'imported') chunksCreated += result.chunks;
|
||||
else if (result.status === 'skipped' && (result as { error?: string }).error) {
|
||||
failedFiles.push({ path: to, error: String((result as { error?: string }).error) });
|
||||
@@ -2910,68 +2901,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
failedFiles.push({ path: to, error: e instanceof Error ? e.message : String(e) });
|
||||
}
|
||||
}
|
||||
// #3056 reconcile: the rename fell back to add semantics, so the row
|
||||
// that still represents the OLD path is the stale half of the rename
|
||||
// (git reported the old path gone; a plain delete of that path would
|
||||
// remove this row). Two safety rails, both from the #3252 review:
|
||||
//
|
||||
// 1. Delete only after the destination demonstrably materialized —
|
||||
// `imported`, or an errorless `skipped` AT the new slug. Identity
|
||||
// dedup can skip against the OLD row (result.slug === oldSlug),
|
||||
// in which case nothing landed at newSlug and deleting the old
|
||||
// row would destroy the only copy.
|
||||
// 2. Locate the stale row POSITIVELY by `source_path = from`, never
|
||||
// by the oldSlug guess — after a collision, a path-derived
|
||||
// fallback slug could name an unrelated (e.g. manually curated)
|
||||
// row. No source_path match → nothing is deleted (this also means
|
||||
// code-strategy imports, which don't populate source_path, fall
|
||||
// back safely to leaving the old row rather than guessing).
|
||||
//
|
||||
// A failed delete records a `<rename:…>` SENTINEL (not an ordinary
|
||||
// path failure): the gate hard-blocks the bookmark, and — unlike a
|
||||
// plain path row — the auto-skip valve can never chronic-skip it after
|
||||
// N attempts, which would advance the bookmark and make a transient
|
||||
// delete outage a permanent duplicate. The sentinel clears through the
|
||||
// ordinary success path once the rename converges on a later run.
|
||||
let reconcileFailed = false;
|
||||
if (!renameApplied && importResult !== undefined) {
|
||||
const destMaterialized = importResult.status === 'imported' ||
|
||||
(importResult.status === 'skipped' && !importResult.error && importResult.slug === newSlug);
|
||||
if (destMaterialized) {
|
||||
try {
|
||||
const staleMap = await engine.resolveSlugsByPaths([from], { sourceId: opts.sourceId ?? DEFAULT_SOURCE_ID });
|
||||
const staleSlug = staleMap.get(from);
|
||||
if (staleSlug !== undefined && staleSlug !== newSlug) {
|
||||
await engine.deletePage(staleSlug, renameOpts);
|
||||
deletedSlugs.add(staleSlug); // never hand a deleted slug to auto-embed
|
||||
serr(` [sync] rename reconciled: removed stale row ${staleSlug} (${from} -> ${to} fell back to add).`);
|
||||
} else if (staleSlug === undefined) {
|
||||
serr(` [sync] rename fallback: no row has source_path ${from}; stale row (if any) left in place.`);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
reconcileFailed = true;
|
||||
failedFiles.push({
|
||||
path: `<rename:${to}>`,
|
||||
error: `rename reconcile failed (stale row for ${from} not removed): ` +
|
||||
`${e instanceof Error ? e.message : String(e)}`,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
serr(
|
||||
` [sync] rename fallback: ${from} -> ${to} did not materialize at ${newSlug} ` +
|
||||
`(import ${importResult.status}); old row left in place.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Converged (cheap rename, clean reconcile, or nothing to reconcile):
|
||||
// clear any `<rename:…>` sentinel a previous failing run recorded.
|
||||
if (!reconcileFailed) succeededPaths.push(`<rename:${to}>`);
|
||||
pagesAffected.push(newSlug);
|
||||
deletedSlugs.delete(newSlug); // #1284: rename landed on a previously-deleted slug → embeddable again
|
||||
// A failed reconcile must NOT checkpoint: banking `to` would make the
|
||||
// resume filter skip this rename on the retry run, turning a transient
|
||||
// delete failure into a permanent duplicate — the exact bug being fixed.
|
||||
if (!reconcileFailed) await markCompleted(to);
|
||||
await markCompleted(to);
|
||||
progress.tick(1, newSlug);
|
||||
}
|
||||
progress.finish();
|
||||
@@ -3430,10 +3362,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
|
||||
if (!gate.advanced) {
|
||||
const codeBreakdown = formatCodeBreakdown(failedFiles);
|
||||
// Two sentinel classes block here: `<head>` (pin ancestry broken) and
|
||||
// `<rename:…>` (#3056 — a rename-reconcile delete failed and advancing
|
||||
// would permanently bank the duplicate). Pick the message by which fired.
|
||||
if (gate.sentinelBlocked && failedFiles.some(f => f.path === '<head>')) {
|
||||
if (gate.sentinelBlocked) {
|
||||
serr(
|
||||
`\nSync blocked: repository history changed during sync (force-push / reset).\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
@@ -3441,12 +3370,6 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
`a commit that doesn't match the indexed tree. Re-run sync to re-pin against ` +
|
||||
`current HEAD.`,
|
||||
);
|
||||
} else if (gate.sentinelBlocked) {
|
||||
serr(
|
||||
`\nSync blocked: a rename left a stale duplicate that could not be removed:\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
`The next 'gbrain sync' retries the reconcile from the same diff.`,
|
||||
);
|
||||
} else {
|
||||
const fileFailCount = failedFiles.filter(f => isSkippablePath(f.path)).length;
|
||||
serr(
|
||||
|
||||
+8
-15
@@ -3,7 +3,6 @@
|
||||
*
|
||||
* Subcommands:
|
||||
* takes <slug> — list takes for a page
|
||||
* takes list — list all active takes (#2079)
|
||||
* takes search "<query>" [--who h] — keyword search across all takes
|
||||
* takes add <slug> ...flags — append a take (markdown + DB)
|
||||
* takes update <slug> --row N ...flags — update mutable fields
|
||||
@@ -130,10 +129,11 @@ function writeBody(path: string, body: string): void {
|
||||
// --- Subcommands ---
|
||||
|
||||
async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
// #2079: slug is optional. `gbrain takes list` (no slug) lists ALL active
|
||||
// takes — CLI parity with the takes_list operation. A leading flag is not
|
||||
// a slug.
|
||||
const slug = args[0] && !args[0].startsWith('-') ? args[0] : undefined;
|
||||
const slug = args[0];
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain takes <slug> [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
const json = flagPresent(args, '--json');
|
||||
const holder = flagValue(args, '--who');
|
||||
const kind = flagValue(args, '--kind') as string | undefined;
|
||||
@@ -153,19 +153,17 @@ async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const scope = slug ?? 'this brain';
|
||||
if (takes.length === 0) {
|
||||
console.log(`No takes on ${scope}.`);
|
||||
console.log(`No takes on ${slug}.`);
|
||||
return;
|
||||
}
|
||||
console.log(`# Takes on ${scope}\n`);
|
||||
console.log(`# Takes on ${slug}\n`);
|
||||
for (const t of takes) {
|
||||
const tag = t.active ? '' : ' [superseded]';
|
||||
const w = Number(t.weight).toFixed(2);
|
||||
const since = t.since_date ?? '';
|
||||
const src = t.source ? ` — ${t.source}` : '';
|
||||
const where = slug ? '' : `${t.page_slug} `;
|
||||
console.log(`${where}#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
console.log(`#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -557,8 +555,6 @@ export async function runTakes(engine: BrainEngine, args: string[]): Promise<voi
|
||||
Subcommands:
|
||||
takes <slug> [--json] [--who h] [--kind k] [--sort weight|since_date|created_at] [--expired]
|
||||
List takes for a page
|
||||
takes list [--json] [--who h] [--kind k] [--sort ...] [--expired]
|
||||
List all active takes across the brain (#2079)
|
||||
takes search "<query>" [--limit N] [--json]
|
||||
Keyword search across all takes
|
||||
takes add <slug> --claim "..." --kind <fact|take|bet|hunch> --who <holder>
|
||||
@@ -588,9 +584,6 @@ Common flags:
|
||||
const rest = args.slice(1);
|
||||
|
||||
switch (sub) {
|
||||
// #2079: `takes list` used to be parsed as page slug "list" and printed
|
||||
// "No takes on list." — reading exactly like an empty takes table.
|
||||
case 'list': return cmdList(engine, rest);
|
||||
case 'search': return cmdSearch(engine, rest);
|
||||
case 'add': return cmdAdd(engine, rest, await resolveTakesSourceId(engine));
|
||||
case 'update': return cmdUpdate(engine, rest, await resolveTakesSourceId(engine));
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
* import it from `../../src/cli.ts`.
|
||||
*
|
||||
* The single ownership site for: (a) folding file-plane API keys
|
||||
* (openai/anthropic/zeroentropy/openrouter/voyage/dashscope/google) into the gateway env, and (b) threading
|
||||
* (openai/anthropic/zeroentropy/openrouter/voyage) into the gateway env, and (b) threading
|
||||
* local-server `*_BASE_URL` env vars into base_urls. Both matter for the
|
||||
* init-time embedding-key probe — without (a) it would false-warn on
|
||||
* config.json-keyed users, and without (b) a live probe could hit the wrong
|
||||
@@ -44,18 +44,6 @@ export function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig {
|
||||
// multimodal/image embeds despite config.json looking complete. process.env
|
||||
// still wins via the later spread.
|
||||
if (c.voyage_api_key) envFromConfig.VOYAGE_API_KEY = c.voyage_api_key;
|
||||
// #3500: same seam for DashScope. The dashscope + dashscope-rerank recipes
|
||||
// require DASHSCOPE_API_KEY, but the config-plane key was never folded, so
|
||||
// daemon/launchd/MCP contexts with no process-env export failed auth
|
||||
// despite config.json looking complete. process.env still wins via the
|
||||
// later spread.
|
||||
if (c.dashscope_api_key) envFromConfig.DASHSCOPE_API_KEY = c.dashscope_api_key;
|
||||
// #3500: same seam for Google Gemini. The google recipe reads
|
||||
// GOOGLE_GENERATIVE_AI_API_KEY; before this fold, the ONLY way to
|
||||
// configure Gemini was exporting that exact env var. (This closes the
|
||||
// deferral noted in src/core/brain-score-recommendations.ts, whose
|
||||
// HOSTED_EMBED_KEY_CONFIG entry lands in the same change.)
|
||||
if (c.google_api_key) envFromConfig.GOOGLE_GENERATIVE_AI_API_KEY = c.google_api_key;
|
||||
// Azure OpenAI (keyless/Entra): fold the non-secret endpoint/deployment + the
|
||||
// Entra opt-in into the gateway env so the azure-openai recipe works in any
|
||||
// shell (incl. non-interactive agent shells). The bearer token is minted at
|
||||
@@ -98,26 +86,11 @@ export function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig {
|
||||
// every gateway op then throws NO_ANTHROPIC_API_KEY. Drop empty-string /
|
||||
// undefined entries before the merge. Only '' and undefined are dropped —
|
||||
// '0' and 'false' are legitimate values and survive.
|
||||
env: buildEnv(envFromConfig),
|
||||
env: {
|
||||
...envFromConfig,
|
||||
...Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, v]) => v !== undefined && v !== ''),
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge config-plane fallbacks with process.env (env wins for keys carrying a
|
||||
* real value — see #1249 note above), then apply the GEMINI_API_KEY alias:
|
||||
* Google's own docs/SDKs export GEMINI_API_KEY, but the google recipe (and
|
||||
* every gateway read site) uses GOOGLE_GENERATIVE_AI_API_KEY. Precedence:
|
||||
* env GOOGLE_GENERATIVE_AI_API_KEY > env GEMINI_API_KEY > config
|
||||
* google_api_key — i.e. the alias is still process-env, so it beats the
|
||||
* config-plane fallback, but never the canonical env name.
|
||||
*/
|
||||
function buildEnv(envFromConfig: Record<string, string>): Record<string, string> {
|
||||
const envReal = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, v]) => v !== undefined && v !== ''),
|
||||
) as Record<string, string>;
|
||||
const merged = { ...envFromConfig, ...envReal };
|
||||
if (!envReal.GOOGLE_GENERATIVE_AI_API_KEY && envReal.GEMINI_API_KEY) {
|
||||
merged.GOOGLE_GENERATIVE_AI_API_KEY = envReal.GEMINI_API_KEY;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
+2
-63
@@ -642,42 +642,8 @@ function warnRecipesMissingBatchTokens(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only reset baseline (#3554). The bunfig preload
|
||||
* (`test/helpers/legacy-embedding-preload.ts`) pins the gateway to the legacy
|
||||
* OpenAI/1536 config at process start, but `resetGateway()` used to wipe that
|
||||
* pin to `_config = null`. The next test file's engine connect then
|
||||
* reconfigured from the SHIPPED default (zembed-1 @ 1280) and every 1536-d
|
||||
* fixture in that file exploded with `expected 1280 dimensions, not 1536` —
|
||||
* a cross-file mine whose placement depended on shard bin-packing.
|
||||
*
|
||||
* When a baseline factory is registered, `resetGateway()` means "back to the
|
||||
* test baseline" instead of "unconfigured": it clears everything as before,
|
||||
* then re-applies the factory's config via `configureGateway()`. A factory
|
||||
* (not a frozen config) so each re-application captures fresh
|
||||
* `process.env`, matching the preload's original `applyLegacy()` semantics.
|
||||
*
|
||||
* Production is untouched: nothing in `src/` calls `resetGateway()` or this
|
||||
* setter, so in production the baseline is never registered and
|
||||
* `resetGateway()` still fully unconfigures. Same `__*ForTests` seam
|
||||
* convention as `__setEmbedTransportForTests` above.
|
||||
*/
|
||||
let _resetBaseline: (() => AIGatewayConfig) | null = null;
|
||||
|
||||
/**
|
||||
* Register (or clear, with `null`) the config factory that `resetGateway()`
|
||||
* re-applies. Called once by the bunfig test preload.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __setGatewayResetBaselineForTests(
|
||||
factory: (() => AIGatewayConfig) | null,
|
||||
): void {
|
||||
_resetBaseline = factory;
|
||||
}
|
||||
|
||||
/** Clear every piece of module state. Shared by both reset flavors. */
|
||||
function clearGatewayState(): void {
|
||||
/** Reset (for tests). */
|
||||
export function resetGateway(): void {
|
||||
_config = null;
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
@@ -689,33 +655,6 @@ function clearGatewayState(): void {
|
||||
_extendedModels.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset (for tests). Clears all module state (config, model cache, shrink
|
||||
* state, transports, warned recipes, extended models), then — if a test
|
||||
* baseline is registered — re-applies it so the gateway returns to the
|
||||
* process-wide test default instead of an unconfigured limbo (#3554).
|
||||
*/
|
||||
export function resetGateway(): void {
|
||||
clearGatewayState();
|
||||
// configureGateway re-clears _modelCache/_shrinkState/_extendedModels and
|
||||
// registers the baseline's models; transports are NOT touched by it, so a
|
||||
// stale test transport can never leak back in through this path.
|
||||
if (_resetBaseline) configureGateway(_resetBaseline());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset AND stay unconfigured, ignoring any registered baseline. For the
|
||||
* handful of tests that assert genuine no-gateway behavior
|
||||
* (`no_gateway_config` diagnosis, `isAvailable() === false`, graceful
|
||||
* degradation paths). The preload's per-test beforeEach restores the
|
||||
* baseline before the next test, so this cannot leak across tests.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __unconfigureGatewayForTests(): void {
|
||||
clearGatewayState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only seam. Replaces the function the gateway calls to embed a
|
||||
* sub-batch. Pass `null` to restore the real `embedMany` from the AI SDK.
|
||||
|
||||
@@ -31,11 +31,6 @@ export const dashscope: Recipe = {
|
||||
// path. Conservative declaration so the gateway pre-splits before
|
||||
// hitting whatever undocumented server-side limit exists.
|
||||
max_batch_tokens: 8192,
|
||||
// DashScope's OpenAI-compat /embeddings endpoint rejects requests with
|
||||
// more than 10 input items (documented Model Studio cap). The gateway's
|
||||
// capBatchItems pre-split enforces this; max_batch_tokens above keeps
|
||||
// guarding aggregate token size. Concept from community PRs #2643/#2405.
|
||||
max_batch_items: 10,
|
||||
// text-embedding-v3 mixes English + CJK heavily; the tokenizer is
|
||||
// closer to Voyage density than OpenAI tiktoken for CJK-dominant
|
||||
// content. Conservative chars_per_token=2 leaves headroom.
|
||||
|
||||
@@ -16,17 +16,6 @@ export const google: Recipe = {
|
||||
dims_options: [768, 1536, 3072],
|
||||
cost_per_1m_tokens_usd: 0.15,
|
||||
price_last_verified: '2026-04-20',
|
||||
// Gemini's embedding endpoint has a low per-request cap relative to
|
||||
// Voyage. Declaring max_batch_tokens makes the gateway pre-split bulk
|
||||
// batches proactively (splitByTokenBudget) instead of relying solely on
|
||||
// the recursive-halving retry on a token-limit rejection. Conservative
|
||||
// value: each gemini-embedding-001 input tops out at 2048 tokens, so a
|
||||
// 20k budget × 0.8 safety keeps a batch well within request limits while
|
||||
// staying efficient. chars_per_token ~4 matches Gemini's SentencePiece
|
||||
// density on English. Tunable; recursion stays the backstop.
|
||||
max_batch_tokens: 20_000,
|
||||
chars_per_token: 4,
|
||||
safety_factor: 0.8,
|
||||
},
|
||||
expansion: {
|
||||
models: ['gemini-2.0-flash', 'gemini-2.0-flash-lite'],
|
||||
@@ -34,14 +23,11 @@ export const google: Recipe = {
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
// gemini-1.5-pro was retired by Google (#3510) — deliberately NOT
|
||||
// listed. Default-slot guard tests validate hardcoded defaults against
|
||||
// this list, so re-adding a dead model here masks dead defaults.
|
||||
models: ['gemini-2.0-flash-exp', 'gemini-2.0-flash'],
|
||||
models: ['gemini-2.0-flash-exp', 'gemini-2.0-flash', 'gemini-1.5-pro'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 1000000, // Gemini 2.0 Flash
|
||||
max_context_tokens: 1000000, // Gemini 1.5 Pro
|
||||
cost_per_1m_input_usd: 0.30,
|
||||
cost_per_1m_output_usd: 1.20,
|
||||
price_last_verified: '2026-04-20',
|
||||
|
||||
@@ -59,24 +59,10 @@ const ALL: Recipe[] = [
|
||||
/** Map from `provider:id` key to recipe. */
|
||||
export const RECIPES: Map<string, Recipe> = new Map(ALL.map(r => [r.id, r]));
|
||||
|
||||
/**
|
||||
* Test-only seam. Synthetic recipes appended to the registry so tests can
|
||||
* exercise registry-walking logic — notably gateway.ts's missing-batch-cap
|
||||
* startup warning — against a recipe that intentionally omits a field,
|
||||
* without editing the shipped `ALL` array. Every real embedding recipe now
|
||||
* declares a cap (token budget, `no_batch_cap`, or item cap), so a synthetic
|
||||
* cap-less recipe is the only way to cover the warn-fires path. Empty in
|
||||
* production (nothing in `src/` calls the setter); pass `[]` to reset.
|
||||
*/
|
||||
let _testRecipes: Recipe[] = [];
|
||||
export function __setTestRecipesForTests(recipes: Recipe[]): void {
|
||||
_testRecipes = recipes;
|
||||
}
|
||||
|
||||
export function getRecipe(id: string): Recipe | undefined {
|
||||
return RECIPES.get(id) ?? _testRecipes.find(r => r.id === id);
|
||||
return RECIPES.get(id);
|
||||
}
|
||||
|
||||
export function listRecipes(): Recipe[] {
|
||||
return _testRecipes.length > 0 ? [...ALL, ..._testRecipes] : [...ALL];
|
||||
return [...ALL];
|
||||
}
|
||||
|
||||
@@ -180,17 +180,6 @@ export const openrouter: Recipe = {
|
||||
// to pre-split batches, NOT per-input. Per-input is enforced upstream.
|
||||
max_batch_tokens: 300_000,
|
||||
},
|
||||
// Expansion uses the same routed OpenAI-compatible language-model endpoint
|
||||
// as chat. Keep a small cheap/fast advisory set; the openai-compat tier
|
||||
// still accepts any user-configured OpenRouter provider/model ID.
|
||||
expansion: {
|
||||
models: [
|
||||
'anthropic/claude-haiku-4.5',
|
||||
'google/gemini-3-flash-preview',
|
||||
'deepseek/deepseek-chat',
|
||||
],
|
||||
price_last_verified: '2026-05-20',
|
||||
},
|
||||
chat: {
|
||||
// Curated entry points (verified against OR's catalog 2026-05-20). The
|
||||
// openai-compat tier does NOT enforce this list at runtime — users can
|
||||
|
||||
@@ -37,10 +37,8 @@ export const voyage: Recipe = {
|
||||
'voyage-multimodal-3',
|
||||
],
|
||||
default_dims: 1024,
|
||||
// Display hint for `gbrain providers` only (billing math goes through
|
||||
// src/core/embedding-pricing.ts). Rate for the default voyage-4-large.
|
||||
cost_per_1m_tokens_usd: 0.12,
|
||||
price_last_verified: '2026-07-28',
|
||||
cost_per_1m_tokens_usd: 0.18,
|
||||
price_last_verified: '2026-04-20',
|
||||
// Voyage enforces 120K tokens per batch. Voyage's tokenizer runs
|
||||
// ~3-4× denser than OpenAI tiktoken on mixed content (code/JSON/CJK),
|
||||
// so the per-recipe pre-split uses 1 char ≈ 1 token at 0.5 utilization
|
||||
|
||||
@@ -37,13 +37,7 @@ export function readRecentParserProbeEvents(
|
||||
days = 7,
|
||||
now: Date = new Date(),
|
||||
): ParserProbeAuditEvent[] {
|
||||
// Chronological order (oldest → newest). The shared reader walks the
|
||||
// CURRENT week's file first, then the previous week's, so without sorting
|
||||
// the array tail is the OLDEST in-window event whenever last week's file
|
||||
// has entries — and doctor's "latest" (which reads the tail) reported a
|
||||
// days-old run while counts included the newest one.
|
||||
return writer.readRecent(days, now)
|
||||
.sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
||||
return writer.readRecent(days, now);
|
||||
}
|
||||
|
||||
/** Exposed for tests pinning the rotation edge cases. */
|
||||
|
||||
@@ -118,10 +118,5 @@ export function readRecentQualityProbeEvents(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Chronological order (oldest → newest). Events accumulate across two
|
||||
// week files read current-week-FIRST, so without sorting the array tail
|
||||
// is the OLDEST in-window event whenever last week's file has entries —
|
||||
// and doctor's "Latest:" (which reads the tail) reported a days-old run
|
||||
// while the counts included the newest one.
|
||||
return out.sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -13,13 +13,17 @@ import { parseModelId } from './ai/model-resolver.ts';
|
||||
*
|
||||
* Only keys that `buildGatewayConfig` (src/core/ai/build-gateway-config.ts)
|
||||
* actually folds from config into the gateway env may appear here.
|
||||
* GOOGLE_GENERATIVE_AI_API_KEY is deliberately absent: its config field is NOT
|
||||
* threaded to the gateway today, so the producer closures fall through to
|
||||
* checking `process.env` ONLY for it. That matches what the gateway can
|
||||
* actually use (the recipe reads that key from env). Counting a config-plane
|
||||
* google_api_key here would be a false positive: doctor/autopilot would call
|
||||
* the provider "configured" and dispatch an embed.stale job that then fails
|
||||
* auth at the gateway. When a future change threads google_api_key into
|
||||
* buildGatewayConfig, re-add the matching entry here in the same change.
|
||||
*
|
||||
* VOYAGE_API_KEY → voyage_api_key was the same kind of gap (#2662) until
|
||||
* buildGatewayConfig started folding it — now safe to list here too.
|
||||
* GOOGLE_GENERATIVE_AI_API_KEY → google_api_key and DASHSCOPE_API_KEY →
|
||||
* dashscope_api_key joined for the same reason (#3500): both are folded by
|
||||
* buildGatewayConfig now, so a config-plane key is genuinely usable by the
|
||||
* gateway and counting it here is no longer a false positive.
|
||||
*
|
||||
* Caveat inherited from the existing OPENAI_API_KEY/ZEROENTROPY_API_KEY
|
||||
* entries (unchanged by #2662, noted here for anyone extending this map):
|
||||
@@ -36,8 +40,6 @@ export const HOSTED_EMBED_KEY_CONFIG: Record<string, string> = {
|
||||
OPENAI_API_KEY: 'openai_api_key',
|
||||
ZEROENTROPY_API_KEY: 'zeroentropy_api_key',
|
||||
VOYAGE_API_KEY: 'voyage_api_key',
|
||||
GOOGLE_GENERATIVE_AI_API_KEY: 'google_api_key',
|
||||
DASHSCOPE_API_KEY: 'dashscope_api_key',
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
+11
-125
@@ -40,7 +40,6 @@ export const LINKABLE_ENTITY_TYPES = ['person', 'company', 'organization', 'enti
|
||||
* types in.
|
||||
*/
|
||||
const MIN_NAME_LENGTH = 4;
|
||||
const MIN_CJK_NAME_LENGTH = 2;
|
||||
|
||||
/**
|
||||
* Built-in ignore list — common ambiguous tokens whose body-text mentions
|
||||
@@ -105,12 +104,12 @@ export interface FindMentionsOpts {
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs.
|
||||
*
|
||||
* ASCII: each `[a-zA-Z0-9]+` run is a single token, lowercased.
|
||||
* CJK: each CJK character (Chinese/Japanese/Korean) is an individual
|
||||
* token, lowercased. This allows the normal maximal-munch scan path
|
||||
* to reach CJK gazetteer entries without a separate substring pass.
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs for every
|
||||
* `[a-zA-Z0-9]+` run, lowercased. Non-ASCII (CJK, accented) is
|
||||
* deliberately not tokenized in v1 — entity gazetteer is English-dominant
|
||||
* in production today. Widening to `\p{L}+` is a future option once a
|
||||
* real CJK entity catalog appears (filed under TODO-1 + a TODO for
|
||||
* Unicode-aware tokenization).
|
||||
*
|
||||
* Possessive "Acme's" tokenizes as ['acme', 's'] (single-quote breaks the
|
||||
* run) — single-word "Acme" lookup succeeds at offset 0; the trailing 's'
|
||||
@@ -128,129 +127,18 @@ function tokenizeForScan(text: string): ScannedToken[] {
|
||||
const out: ScannedToken[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
|
||||
// Collect ASCII token spans first.
|
||||
const asciiSpans: Array<{ start: number; end: number }> = [];
|
||||
while ((m = TOKEN_RE.exec(text)) !== null) {
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length });
|
||||
}
|
||||
|
||||
// Walk character-by-character: emit ASCII tokens at their start positions,
|
||||
// then emit individual CJK characters for non-ASCII positions that fall
|
||||
// outside ASCII token spans.
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < text.length;) {
|
||||
const cp = text.codePointAt(i) ?? 0;
|
||||
const isCJK = (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af);
|
||||
|
||||
// Advance asciiIdx past any spans that end before or at i.
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) {
|
||||
asciiIdx++;
|
||||
}
|
||||
|
||||
// If position i is inside an ASCII token span, emit the full ASCII token
|
||||
// and jump past it.
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
const span = asciiSpans[asciiIdx]!;
|
||||
const token = text.slice(span.start, span.end);
|
||||
out.push({ text: token.toLowerCase(), offset: span.start, length: token.length });
|
||||
i = span.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// CJK: emit as individual character token.
|
||||
if (isCJK) {
|
||||
const charLen = cp > 0xffff ? 2 : 1; // surrogate pair
|
||||
const charStr = text.slice(i, i + charLen);
|
||||
out.push({ text: charStr.toLowerCase(), offset: i, length: charLen });
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
out.push({ text: m[0].toLowerCase(), offset: m.index, length: m[0].length });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function hasCJK(s: string): boolean {
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function cjkCharCount(s: string): number {
|
||||
let count = 0;
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a page title for gazetteer insertion.
|
||||
*
|
||||
* ASCII titles: standard `[a-zA-Z0-9]+` tokenization, lowercased.
|
||||
* CJK titles (no ASCII content): split into individual characters —
|
||||
* e.g. "纳瓦尔" → ["纳","瓦","尔"]. This allows normal multi-token
|
||||
* maximal-munch matching to work with character-level CJK tokens
|
||||
* produced by `tokenizeForScan`.
|
||||
* Mixed CJK+ASCII titles: ASCII parts tokenized normally, CJK parts
|
||||
* split into individual characters.
|
||||
*/
|
||||
function tokenizeTitle(title: string): string[] {
|
||||
const tokens: string[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
const hasAscii = TOKEN_RE.test(title);
|
||||
if (hasAscii) {
|
||||
// Mixed ASCII+CJK or pure ASCII: tokenize ASCII normally, then
|
||||
// append individual CJK characters in order.
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
const asciiSpans: Array<{ start: number; end: number; text: string }> = [];
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) {
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length, text: m[0].toLowerCase() });
|
||||
}
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < title.length;) {
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) asciiIdx++;
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
tokens.push(asciiSpans[asciiIdx]!.text);
|
||||
i = asciiSpans[asciiIdx]!.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
if (hasCJK(title[i]!)) {
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Pure CJK (no ASCII content): split into individual characters.
|
||||
if (hasCJK(title)) {
|
||||
for (let i = 0; i < title.length;) {
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Non-ASCII, non-CJK title (emoji, symbols, etc.) — empty set.
|
||||
return [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) tokens.push(m[0].toLowerCase());
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -287,9 +175,7 @@ export async function buildGazetteer(
|
||||
|
||||
const gazetteer: Gazetteer = new Map();
|
||||
for (const row of rows) {
|
||||
if (!row.title) continue;
|
||||
if (!hasCJK(row.title) && row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (hasCJK(row.title) && cjkCharCount(row.title) < MIN_CJK_NAME_LENGTH) continue;
|
||||
if (!row.title || row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (ignoreSet.has(row.title) && !existingTitles.has(row.title)) continue;
|
||||
|
||||
const tokens = tokenizeTitle(row.title);
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
|
||||
import { chunkText as recursiveChunk } from './recursive.ts';
|
||||
import { buildQualifiedName } from './qualified-names.ts';
|
||||
import { CJK_SLUG_CHARS, CJK_RANGES_REGEX } from '../cjk.ts';
|
||||
|
||||
// Embed the tree-sitter runtime + per-language grammars as files.
|
||||
// `with { type: 'file' }` returns a path (string) at runtime. Bun bundles
|
||||
@@ -717,7 +716,7 @@ export async function chunkCodeTextFull(
|
||||
}
|
||||
|
||||
if (chunks.length === 0) {
|
||||
return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
|
||||
return { chunks: capOversizedChunks(fallbackChunks(source, filePath, language, opts), filePath, language, opts), edges: rawEdges };
|
||||
}
|
||||
return { chunks: capOversizedChunks(mergeSmallSiblings(chunks, chunkTarget), filePath, language, opts), edges: rawEdges };
|
||||
} catch {
|
||||
@@ -843,10 +842,10 @@ function capOversizedChunks(
|
||||
opts: CodeChunkOptions,
|
||||
): CodeChunk[] {
|
||||
const cap = opts.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS;
|
||||
if (!chunks.some((c) => estimateEmbedTokens(c.text) > cap)) return chunks;
|
||||
if (!chunks.some((c) => estimateTokens(c.text) > cap)) return chunks;
|
||||
const out: CodeChunk[] = [];
|
||||
for (const c of chunks) {
|
||||
if (estimateEmbedTokens(c.text) <= cap) {
|
||||
if (estimateTokens(c.text) <= cap) {
|
||||
out.push({ ...c, index: out.length });
|
||||
continue;
|
||||
}
|
||||
@@ -881,43 +880,17 @@ function splitToTokenBudget(text: string, cap: number, opts: CodeChunkOptions):
|
||||
chunkOverlap: opts.fallbackOverlapWords ?? 50,
|
||||
}).map((p) => p.text);
|
||||
for (const piece of pieces) {
|
||||
if (estimateEmbedTokens(piece) <= cap) {
|
||||
if (estimateTokens(piece) <= cap) {
|
||||
out.push(piece);
|
||||
continue;
|
||||
}
|
||||
// Hard-split slice size. Pure-ASCII pieces: ~3.5 chars/token is a
|
||||
// conservative cl100k estimate for source text. CJK-containing pieces:
|
||||
// the weighted estimate can reach 1 token/char, so budget 1 char/token
|
||||
// to keep every slice under cap by construction.
|
||||
const charBudget = Math.max(1, Math.floor(cap * (CJK_RANGES_REGEX.test(piece) ? 1 : 3.5)));
|
||||
// ~3.5 chars/token is a conservative cl100k estimate for source text.
|
||||
const charBudget = Math.max(1, Math.floor(cap * 3.5));
|
||||
for (let i = 0; i < piece.length; i += charBudget) out.push(piece.slice(i, i + charBudget));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const CJK_CHARS_G = new RegExp(`[${CJK_SLUG_CHARS}]`, 'g');
|
||||
|
||||
/**
|
||||
* Embedding-safe token estimate for the oversize cap. estimateTokens
|
||||
* (cl100k) matches embedding-family tokenizers closely on pure-ASCII source
|
||||
* (measured identical on English prose and JSON vs Qwen3-Embedding), but
|
||||
* UNDERCOUNTS mixed CJK+ASCII chunks — measured −31% on URL-dense Korean
|
||||
* text vs the Qwen3 embedding tokenizer, which is exactly the shape that
|
||||
* overflows strict embedding backends (#2826). For chunks containing CJK,
|
||||
* take the max of cl100k and a per-char-class overestimate (CJK 1.0/char,
|
||||
* other non-whitespace 0.75/char, whitespace 0.1/char). CJK-DOMINANT text
|
||||
* is unaffected too: cl100k already counts it above the weighted form, so
|
||||
* max() returns the same value as today. Only mixed-script chunks — the
|
||||
* measured divergence class — estimate higher.
|
||||
*/
|
||||
export function estimateEmbedTokens(text: string): number {
|
||||
const cjk = (text.match(CJK_CHARS_G) || []).length;
|
||||
if (cjk === 0) return estimateTokens(text);
|
||||
const ws = (text.match(/\s/g) || []).length;
|
||||
const weighted = Math.ceil(cjk + (text.length - cjk - ws) * 0.75 + ws * 0.1);
|
||||
return Math.max(estimateTokens(text), weighted);
|
||||
}
|
||||
|
||||
// ---------- Internals ----------
|
||||
|
||||
function fallbackChunks(
|
||||
@@ -928,7 +901,7 @@ function fallbackChunks(
|
||||
): CodeChunk[] {
|
||||
const size = opts.fallbackChunkSizeWords ?? 300;
|
||||
const overlap = opts.fallbackOverlapWords ?? 50;
|
||||
const chunks = recursiveChunk(source, { chunkSize: size, chunkOverlap: overlap }).map((chunk, index) =>
|
||||
return recursiveChunk(source, { chunkSize: size, chunkOverlap: overlap }).map((chunk, index) =>
|
||||
buildChunk({
|
||||
body: chunk.text, filePath, language,
|
||||
symbolName: null, symbolType: 'module',
|
||||
@@ -936,14 +909,6 @@ function fallbackChunks(
|
||||
index,
|
||||
}),
|
||||
);
|
||||
// Route every fallback emission through the oversize net. Previously only
|
||||
// the empty-AST branch wrapped its fallback in capOversizedChunks — the
|
||||
// no-language, parse-timeout, no-semantic-nodes (every JSON/YAML fence:
|
||||
// their node types aren't in TOP_LEVEL_TYPES) and parse-throw branches
|
||||
// shipped word-counted chunks unchecked, and the word pipeline undercounts
|
||||
// exactly the dense content (JSON, minified, CJK-mixed) that overflows
|
||||
// embedders. Hoisting the cap here covers all five paths at once.
|
||||
return capOversizedChunks(chunks, filePath, language, opts);
|
||||
}
|
||||
|
||||
function buildChunk(input: {
|
||||
|
||||
+5
-28
@@ -21,35 +21,12 @@ export const CJK_SLUG_CHARS = '一-鿿-ゟ゠-ヿ가-';
|
||||
export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`);
|
||||
|
||||
/**
|
||||
* Slug "word" character class (#3417): every script's letters, not just
|
||||
* Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any
|
||||
* regex composed from this string (without `u`, `\p{Ll}` silently matches
|
||||
* the literal chars `p`, `L`, `l`, `{`, `}`).
|
||||
*
|
||||
* \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …)
|
||||
* \p{Lm} modifier letters
|
||||
* \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …)
|
||||
* \p{M} combining marks that survive the Latin accent-strip pass
|
||||
* (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs)
|
||||
* \p{N} numbers (0-9, Arabic-Indic digits, …)
|
||||
*
|
||||
* Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment()
|
||||
* lowercases before filtering, so validators stay lowercase-canonical.
|
||||
*
|
||||
* Distinct from CJK_SLUG_CHARS above, which also drives the
|
||||
* countCJKAwareWords density heuristic — do NOT merge the two, or slug
|
||||
* grammar changes silently change chunking behavior.
|
||||
* Page-slug segment grammar (no anchors): alnum-or-CJK lead char, then
|
||||
* alnum/CJK/hyphen continuation. Single source for validatePageSlug
|
||||
* (operations.ts), SlugRegistry's SLUG_RE, and the dream-cycle
|
||||
* SUMMARY_SLUG_RE so every slug validator shares one grammar (#738).
|
||||
*/
|
||||
export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}';
|
||||
|
||||
/**
|
||||
* Page-slug segment grammar (no anchors): word-char lead, then word-char or
|
||||
* hyphen continuation. Single source for validatePageSlug (operations.ts),
|
||||
* SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug
|
||||
* validator shares one grammar (#738). Compose with the `u` flag — see
|
||||
* SLUG_WORD_CHARS.
|
||||
*/
|
||||
export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`;
|
||||
export const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`;
|
||||
|
||||
export const CJK_SENTENCE_DELIMITERS = ['。', '!', '?']; // 。!?
|
||||
export const CJK_CLAUSE_DELIMITERS = [';', ':', ',', '、']; // ;:,、
|
||||
|
||||
+20
-72
@@ -51,29 +51,9 @@ export const DEFAULT_CLI_OPTIONS: CliOptions = {
|
||||
*
|
||||
* Unknown flags are passed through unchanged — per-command parsers see them.
|
||||
*/
|
||||
/**
|
||||
* #3013: commands that parse their own `--timeout` flag out of argv.
|
||||
* `sync` reads a seconds-based graceful-abort budget (src/commands/sync.ts +
|
||||
* resolveSyncHardDeadline); `remote` reads a ms-based request budget
|
||||
* (src/commands/remote.ts). For these commands the global parser must hand
|
||||
* the flag back: claiming it stripped the flag before the per-command parser
|
||||
* could read it, and — for `sync` — a non-null global timeoutMs flipped the
|
||||
* read-only dispatch gate in cli.ts, rerouting a write command into
|
||||
* dispatchReadOnlyCommand (exit 1 before any work ran).
|
||||
*/
|
||||
export const TIMEOUT_OWNING_COMMANDS = new Set(['sync', 'remote']);
|
||||
|
||||
export function parseGlobalFlags(argv: string[]): { cliOpts: CliOptions; rest: string[] } {
|
||||
const cliOpts: CliOptions = { ...DEFAULT_CLI_OPTIONS };
|
||||
// #3013: --timeout can't be resolved inline — whether the GLOBAL parser
|
||||
// claims it depends on which command is running, and the command token is
|
||||
// only known once the whole argv has been scanned (global flags may precede
|
||||
// it). The scan collects positional slots; --timeout slots are resolved in
|
||||
// a second pass below.
|
||||
type Slot =
|
||||
| { plain: string }
|
||||
| { timeoutValue: string; equalsForm: boolean };
|
||||
const slots: Slot[] = [];
|
||||
const rest: string[] = [];
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
@@ -94,7 +74,7 @@ export function parseGlobalFlags(argv: string[]): { cliOpts: CliOptions; rest: s
|
||||
continue;
|
||||
}
|
||||
// not a number — let per-command parser handle; pass through
|
||||
slots.push({ plain: a });
|
||||
rest.push(a);
|
||||
continue;
|
||||
}
|
||||
if (a.startsWith('--progress-interval=')) {
|
||||
@@ -104,20 +84,29 @@ export function parseGlobalFlags(argv: string[]): { cliOpts: CliOptions; rest: s
|
||||
cliOpts.progressInterval = parsed;
|
||||
continue;
|
||||
}
|
||||
slots.push({ plain: a });
|
||||
rest.push(a);
|
||||
continue;
|
||||
}
|
||||
// v0.31.1: --timeout=Ns or --timeout Ns. Accepts plain ms, "30s", "2m".
|
||||
// A following token that is itself a flag is NOT a value — leave it for
|
||||
// its own iteration (pre-#3013 behavior: an unparseable next token was
|
||||
// never consumed).
|
||||
if (a === '--timeout' && i + 1 < argv.length && !argv[i + 1].startsWith('-')) {
|
||||
slots.push({ timeoutValue: argv[i + 1], equalsForm: false });
|
||||
i++;
|
||||
if (a === '--timeout' && i + 1 < argv.length) {
|
||||
const next = argv[i + 1];
|
||||
const parsed = parseTimeout(next);
|
||||
if (parsed !== null) {
|
||||
cliOpts.timeoutMs = parsed;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
rest.push(a);
|
||||
continue;
|
||||
}
|
||||
if (a.startsWith('--timeout=')) {
|
||||
slots.push({ timeoutValue: a.slice('--timeout='.length), equalsForm: true });
|
||||
const val = a.slice('--timeout='.length);
|
||||
const parsed = parseTimeout(val);
|
||||
if (parsed !== null) {
|
||||
cliOpts.timeoutMs = parsed;
|
||||
continue;
|
||||
}
|
||||
rest.push(a);
|
||||
continue;
|
||||
}
|
||||
// v0.40.4 — --explain for `gbrain search/query` per-stage attribution.
|
||||
@@ -125,50 +114,9 @@ export function parseGlobalFlags(argv: string[]): { cliOpts: CliOptions; rest: s
|
||||
cliOpts.explain = true;
|
||||
continue;
|
||||
}
|
||||
slots.push({ plain: a });
|
||||
rest.push(a);
|
||||
}
|
||||
|
||||
// The command is the first plain token (matches `command = rest[0]` in
|
||||
// cli.ts). If it owns --timeout, every --timeout is handed back in the
|
||||
// space-separated spelling (the only form the owning parsers read; this
|
||||
// also normalizes `--timeout=60s`), value verbatim so the owning command
|
||||
// applies its own unit + validity rules (`sync`: bare integers are
|
||||
// SECONDS, `ms`/fractional rejected loudly; `remote` accepts `h`).
|
||||
// Handed-back flags are APPENDED after every other token: both owning
|
||||
// commands treat leading args as positional subcommands (`sync trigger`,
|
||||
// `remote ping`) and locate --timeout by scanning args, so appending can't
|
||||
// shadow a subcommand while duplicate flags keep their argv order (the
|
||||
// owning parsers' first-occurrence-wins precedence matches what the user
|
||||
// typed). Non-owning commands keep the pre-#3013 global behavior:
|
||||
// parseable values are claimed into cliOpts.timeoutMs (last one wins),
|
||||
// unparseable ones pass through in their original spelling for the
|
||||
// per-command parser.
|
||||
const commandSlot = slots.find((s): s is { plain: string } => 'plain' in s);
|
||||
const commandOwnsTimeout =
|
||||
commandSlot !== undefined && TIMEOUT_OWNING_COMMANDS.has(commandSlot.plain);
|
||||
|
||||
const rest: string[] = [];
|
||||
const handback: string[] = [];
|
||||
for (const s of slots) {
|
||||
if ('plain' in s) {
|
||||
rest.push(s.plain);
|
||||
continue;
|
||||
}
|
||||
if (commandOwnsTimeout) {
|
||||
handback.push('--timeout', s.timeoutValue);
|
||||
continue;
|
||||
}
|
||||
const parsed = parseTimeout(s.timeoutValue);
|
||||
if (parsed !== null) {
|
||||
cliOpts.timeoutMs = parsed;
|
||||
} else if (s.equalsForm) {
|
||||
rest.push(`--timeout=${s.timeoutValue}`);
|
||||
} else {
|
||||
rest.push('--timeout', s.timeoutValue);
|
||||
}
|
||||
}
|
||||
rest.push(...handback);
|
||||
|
||||
return { cliOpts, rest };
|
||||
}
|
||||
|
||||
|
||||
@@ -63,23 +63,6 @@ export interface GBrainConfig {
|
||||
* config.json file-plane route is wired through today.
|
||||
*/
|
||||
voyage_api_key?: string;
|
||||
/**
|
||||
* Alibaba DashScope API key (#3500). File-plane slot so config.json's
|
||||
* `dashscope_api_key` reaches the dashscope / dashscope-rerank recipes:
|
||||
* file plane → buildGatewayConfig env dict → recipe reads
|
||||
* DASHSCOPE_API_KEY. Same fold pattern (and same DB-plane caveat) as
|
||||
* voyage_api_key above.
|
||||
*/
|
||||
dashscope_api_key?: string;
|
||||
/**
|
||||
* Google Gemini API key (#3500). File-plane slot folded into the gateway
|
||||
* env as GOOGLE_GENERATIVE_AI_API_KEY (the name the google recipe reads).
|
||||
* buildGatewayConfig also accepts process-env GEMINI_API_KEY — the name
|
||||
* Google's own docs/SDKs use — as an alias for
|
||||
* GOOGLE_GENERATIVE_AI_API_KEY. Same fold pattern (and same DB-plane
|
||||
* caveat) as voyage_api_key above.
|
||||
*/
|
||||
google_api_key?: string;
|
||||
/** Azure OpenAI (keyless/Entra). Non-secret endpoint + deployment + Entra opt-in,
|
||||
* folded into the gateway env so the azure-openai recipe works in any shell.
|
||||
* The bearer token is minted at request time via `az` — no secret stored here. */
|
||||
@@ -936,8 +919,6 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
|
||||
'zeroentropy_api_key',
|
||||
'openrouter_api_key',
|
||||
'voyage_api_key',
|
||||
'dashscope_api_key',
|
||||
'google_api_key',
|
||||
'azure_openai_endpoint',
|
||||
'azure_openai_deployment',
|
||||
'azure_openai_use_entra',
|
||||
|
||||
@@ -145,19 +145,6 @@ export function computeCorpusGeneration(args: {
|
||||
return h.digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — the corpus_generation a page lands on when a plain re-embed path
|
||||
* (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
|
||||
* title-only tier (the D14 fallback tier; synopsis re-generation is a paid
|
||||
* backfill concern). Callers restamp
|
||||
* `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
|
||||
* so the stamped mode keeps describing the vectors actually in the column.
|
||||
* Matches what the inline import path writes for its title-tier pages.
|
||||
*/
|
||||
export function titleTierCorpusGeneration(): string {
|
||||
return computeCorpusGeneration({ crMode: 'title', haikuModel: DEFAULT_HAIKU_MODEL });
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute source_text_hash for D27 P1-4 cache key composition. The
|
||||
* synopsis cache invalidates correctly when adjacent text changes (page
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
/**
|
||||
* Block-format conversation normalizer.
|
||||
*
|
||||
* Some chat exports — notably the Slack collector gbrain's own ingestion
|
||||
* uses — emit a HEADER + indented-body BLOCK per message instead of the
|
||||
* single-line `**Name** (time): body` shape the built-in patterns
|
||||
* (`builtins.ts`) recognize:
|
||||
*
|
||||
* - **Theo** (Mon 11:18)
|
||||
* Hey everyone — quick update on the renewal.
|
||||
*
|
||||
* Second paragraph of the same message.
|
||||
* - **Juan** (Mon 11:20)
|
||||
* Reply body...
|
||||
*
|
||||
* None of the 14 line-oriented built-ins match this: a leading `- ` list
|
||||
* marker, a day-of-week + time with no trailing colon, and the message body on
|
||||
* the following indented lines. Result: `phase: 'no_match'`, zero messages,
|
||||
* and the whole comms corpus is silently un-extractable (facts stay empty →
|
||||
* `find_trajectory` returns nothing).
|
||||
*
|
||||
* This collapses each block into the canonical `**Name** (HH:MM): <body joined
|
||||
* to one line>` shape so the existing `bold-paren-time` pattern matches; the
|
||||
* per-message date fills in downstream via `fallbackDate` (the page date).
|
||||
*
|
||||
* STRICT no-op unless the block signature is present: the header regex requires
|
||||
* the paren-group to END the line (no inline `: body`), which is exactly what
|
||||
* the single-line patterns always produce — so feeding already-canonical
|
||||
* content through this function returns it unchanged.
|
||||
*/
|
||||
|
||||
// `- **Name** (Mon 11:18)` / `- **Name** (11:18 AM)` / `- **Name** (16:36)`.
|
||||
// Day-of-week optional; 12h/24h time; optional am/pm; the line ENDS at the
|
||||
// close paren (no inline `: body` — that is what distinguishes a block header
|
||||
// from the single-line `**Name** (time): body` patterns).
|
||||
const BLOCK_HEADER =
|
||||
/^\s*-\s+\*\*(.+?)\*\*\s+\((?:[A-Za-z]{2,9}\.?\s+)?(\d{1,2}):(\d{2})(?::\d{2})?\s*([AaPp][Mm])?\)\s*$/;
|
||||
|
||||
/** True when at least one line is a block-format message header. */
|
||||
export function looksLikeBlockConversation(body: string): boolean {
|
||||
for (const line of body.split('\n')) {
|
||||
if (BLOCK_HEADER.test(line)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function to24h(hour: number, ampm?: string): number {
|
||||
if (!ampm) return hour;
|
||||
const lower = ampm.toLowerCase();
|
||||
if (lower === 'pm' && hour < 12) return hour + 12;
|
||||
if (lower === 'am' && hour === 12) return 0;
|
||||
return hour;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse block-format messages into canonical single-line `**Name** (HH:MM):
|
||||
* body` lines. Returns `body` unchanged when no block header is present.
|
||||
*/
|
||||
export function normalizeBlockConversation(body: string): string {
|
||||
if (!looksLikeBlockConversation(body)) return body;
|
||||
|
||||
const lines = body.split('\n');
|
||||
const out: string[] = [];
|
||||
let current: { name: string; time: string } | null = null;
|
||||
let bodyParts: string[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (current) {
|
||||
const text = bodyParts.join(' ').replace(/\s+/g, ' ').trim();
|
||||
out.push(`**${current.name}** (${current.time}): ${text}`);
|
||||
}
|
||||
current = null;
|
||||
bodyParts = [];
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const m = BLOCK_HEADER.exec(line);
|
||||
if (m) {
|
||||
flush();
|
||||
const hour = to24h(parseInt(m[2], 10), m[4]);
|
||||
const time = `${String(hour).padStart(2, '0')}:${m[3]}`;
|
||||
current = { name: m[1].trim(), time };
|
||||
} else if (current) {
|
||||
// Body line of the current message. Drop blank lines; keep the rest.
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) bodyParts.push(trimmed);
|
||||
}
|
||||
// Lines before the first header (page title, leading blanks) are dropped —
|
||||
// they never matched a pattern anyway.
|
||||
}
|
||||
flush();
|
||||
|
||||
return out.length > 0 ? out.join('\n') : body;
|
||||
}
|
||||
@@ -29,7 +29,6 @@ import {
|
||||
BUILTIN_PATTERNS,
|
||||
cleanSpeaker,
|
||||
} from './builtins.ts';
|
||||
import { normalizeBlockConversation } from './normalize-block.ts';
|
||||
import type {
|
||||
DateContext,
|
||||
MatchedMessage,
|
||||
@@ -474,12 +473,6 @@ export function parseConversation(
|
||||
return { messages: [], phase: 'no_match' };
|
||||
}
|
||||
|
||||
// Pre-pass: collapse block-format chat exports (header + indented body, e.g.
|
||||
// the Slack collector's `- **Name** (Mon 11:18)\n body…`) into the canonical
|
||||
// single-line shape the built-in patterns recognize. Strict no-op when no
|
||||
// block header is present, so already-canonical content is untouched.
|
||||
body = normalizeBlockConversation(body);
|
||||
|
||||
const dateCtx = deriveDateContext(opts);
|
||||
|
||||
// Assemble candidate pool: built-ins (minus disabled) + user patterns.
|
||||
|
||||
@@ -51,11 +51,7 @@ export const DEFAULT_SLOTS: SlotConfig[] = [
|
||||
// 2-model quorum without a Google key (verdict: permanently inconclusive).
|
||||
{ id: 'A', model: 'openai:gpt-5.2' },
|
||||
{ id: 'B', model: 'anthropic:claude-opus-4-7' },
|
||||
// gemini-1.5-pro was retired by Google (#3510), so slot C failed even with
|
||||
// a Google key configured. deepseek:deepseek-v4-pro preserves the
|
||||
// three-distinct-provider contract with a model registered in both the
|
||||
// recipe and canonical pricing tables (same replacement as PR #3501).
|
||||
{ id: 'C', model: 'deepseek:deepseek-v4-pro' },
|
||||
{ id: 'C', model: 'google:gemini-1.5-pro' },
|
||||
];
|
||||
|
||||
export interface SlotConfig {
|
||||
|
||||
+3
-14
@@ -895,17 +895,8 @@ export async function resolveSourceForDir(
|
||||
// (the cycleSourceId precedence) or 'default'.
|
||||
if (brainDir === null) return undefined;
|
||||
try {
|
||||
// #2540: exclude archived rows (dream's --source guard refuses to stamp
|
||||
// them, so an archived alias winning here means the stamp silently never
|
||||
// lands and doctor's cycle_freshness stays red on a healthy install) and
|
||||
// order deterministically so a duplicate registration of the same path
|
||||
// can't shadow the active source on whichever row the engine scans first.
|
||||
// Ordering matches listAllSources/sources-ops for operator-output parity.
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM sources
|
||||
WHERE local_path = $1 AND archived = false
|
||||
ORDER BY (id = 'default') DESC, id
|
||||
LIMIT 1`,
|
||||
`SELECT id FROM sources WHERE local_path = $1 LIMIT 1`,
|
||||
[brainDir],
|
||||
);
|
||||
if (rows[0]) return rows[0].id;
|
||||
@@ -1188,9 +1179,7 @@ async function runPhaseExtractFacts(
|
||||
summary: `extract_facts skipped: ${result.legacyRowsPending} legacy v0.31 facts pending fence backfill`,
|
||||
details: {
|
||||
legacyRowsPending: result.legacyRowsPending,
|
||||
// A bare `apply-migrations --yes` no-ops once the v0.32.2 ledger
|
||||
// entry is complete; the retry marker is what re-runs Phase B.
|
||||
hint: 'gbrain apply-migrations --force-retry 0.32.2 && gbrain apply-migrations --yes',
|
||||
hint: 'gbrain apply-migrations --yes',
|
||||
warnings: result.warnings,
|
||||
},
|
||||
};
|
||||
@@ -2441,7 +2430,7 @@ export async function runCycle(
|
||||
try {
|
||||
const { runSchemaSuggestPhase } = await import('./cycle/schema-suggest.ts');
|
||||
const { result, duration_ms } = await timePhase(async () => {
|
||||
const r = await runSchemaSuggestPhase(engine, { sourceId: cycleSourceId, dryRun: !!opts.dryRun });
|
||||
const r = await runSchemaSuggestPhase(engine, { dryRun: !!opts.dryRun });
|
||||
return {
|
||||
phase: 'schema-suggest' as const,
|
||||
status: (r.skipped ? 'skipped' : 'ok') as PhaseStatus,
|
||||
|
||||
@@ -482,52 +482,24 @@ export async function runPhaseExtractAtoms(
|
||||
}
|
||||
|
||||
// 3. Dual-source merge: transcripts + pages, dedup by contentHash.
|
||||
// Transcripts win on COLLISION (origin attribution stays with the raw
|
||||
// transcript file even if the same content was later imported as a
|
||||
// brain page) — that's decided by the two loops below, which register
|
||||
// every transcript hash into `seenHashes` before any page is checked,
|
||||
// same as before this fix. It's independent of the FINAL work-item
|
||||
// ORDER built after them.
|
||||
//
|
||||
// Order is page-item-first, interleaved 1-for-1 with transcripts (NOT
|
||||
// concatenated transcripts-then-pages). The per-call budget cap (step
|
||||
// 4 below) stops processing `work` in list order once
|
||||
// budgetTracker.totalSpent >= budgetCap, skipping everything after
|
||||
// that point. Two failure modes this avoids:
|
||||
// - Concatenation (old code): a transcript corpus that alone
|
||||
// exceeds the budget cap starves the page pool completely, no
|
||||
// matter how many drain batches run.
|
||||
// - Interleaving with transcripts first: still starves ALL pages
|
||||
// whenever the budget only covers exactly one call (item 0 is a
|
||||
// transcript, item 1 — the first page — never gets attempted).
|
||||
// Pages are the ONLY pool `countExtractAtomsBacklog`/doctor's
|
||||
// extract_atoms_backlog check measures (see that function's
|
||||
// docstring), so page-first guarantees the doctor-visible backlog
|
||||
// makes forward progress on every budget-capped call, however tight
|
||||
// the cap — `--drain` can no longer report the same backlog number
|
||||
// forever while atoms keep getting extracted from transcripts.
|
||||
// Transcripts win on collision (origin attribution stays with the
|
||||
// raw transcript file even if the same content was later imported
|
||||
// as a brain page).
|
||||
type WorkItem =
|
||||
| { kind: 'transcript'; filePath: string; content: string; contentHash: string }
|
||||
| { kind: 'page'; slug: string; content: string; contentHash: string };
|
||||
|
||||
const seenHashes = new Set<string>();
|
||||
const transcriptItems: WorkItem[] = [];
|
||||
const work: WorkItem[] = [];
|
||||
for (const t of transcriptsLive) {
|
||||
if (seenHashes.has(t.contentHash)) { duplicatesSkipped++; continue; }
|
||||
seenHashes.add(t.contentHash);
|
||||
transcriptItems.push({ kind: 'transcript', ...t });
|
||||
work.push({ kind: 'transcript', ...t });
|
||||
}
|
||||
const pageItems: WorkItem[] = [];
|
||||
for (const p of pages) {
|
||||
if (seenHashes.has(p.contentHash)) { duplicatesSkipped++; continue; }
|
||||
seenHashes.add(p.contentHash);
|
||||
pageItems.push({ kind: 'page', ...p });
|
||||
}
|
||||
const work: WorkItem[] = [];
|
||||
const maxPoolLen = Math.max(transcriptItems.length, pageItems.length);
|
||||
for (let i = 0; i < maxPoolLen; i++) {
|
||||
if (i < pageItems.length) work.push(pageItems[i]);
|
||||
if (i < transcriptItems.length) work.push(transcriptItems[i]);
|
||||
work.push({ kind: 'page', ...p });
|
||||
}
|
||||
|
||||
// Phase-level no-op: nothing to extract today.
|
||||
|
||||
@@ -17,26 +17,20 @@
|
||||
* DB rows need cleanup (#1781 — the unconditional wipe-and-reinsert
|
||||
* made every cycle non-idempotent, re-appending duplicate rows).
|
||||
*
|
||||
* After the phase, the DB index for every cleanly parsed affected page
|
||||
* matches the fence's canonical (claim, source) row set (modulo embeddings
|
||||
* + runtime-derived fields). Warning-bearing parses are non-authoritative
|
||||
* and preserve that page's existing index. Pages with no fence wipe DB rows
|
||||
* for that page coordinate only; legacy NULL-source_markdown_slug rows
|
||||
* survive because deleteFactsForPage targets source_markdown_slug = slug only.
|
||||
* After the phase, the DB index for every affected page matches the
|
||||
* fence's canonical (claim, source) row set (modulo embeddings +
|
||||
* runtime-derived fields). Pages with no fence wipe DB rows for that
|
||||
* page coordinate only; legacy NULL-source_markdown_slug rows survive
|
||||
* because deleteFactsForPage targets source_markdown_slug = slug only.
|
||||
*
|
||||
* Empty-fence guard (Codex R2-#7; #2484; #2646): the phase refuses to do
|
||||
* its destructive reconciliation pass when genuinely-backfillable legacy
|
||||
* rows still exist — in THIS run's source only (`source_id = sourceId`;
|
||||
* a pending row in source A must not jam extraction for source B — the
|
||||
* source-isolation invariant) — `row_num IS NULL` (never fenced) AND
|
||||
* `entity_slug` resolves to a live page in this source (so the v0_32_2
|
||||
* migration's Phase B could fence them) AND the row is not soft-expired
|
||||
* (`expired_at IS NULL`). Status returns `warn` with a hint to re-run
|
||||
* the v0.32.2 fence backfill (`apply-migrations --force-retry 0.32.2`
|
||||
* then `--yes` — a bare `--yes` is a no-op once the ledger says
|
||||
* complete). Without the guard, an interrupted upgrade where v0_32_2
|
||||
* hasn't run could leave the cycle silently misreporting "0 facts on
|
||||
* people/alice" while legacy rows linger.
|
||||
* Empty-fence guard (Codex R2-#7; #2484): the phase refuses to do its
|
||||
* destructive reconciliation pass when genuinely-backfillable legacy
|
||||
* rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug`
|
||||
* resolves to a live page in this source (so the v0_32_2 migration's
|
||||
* Phase B could fence them). Status returns `warn` with a hint to run
|
||||
* `gbrain apply-migrations --yes`. Without the guard, an interrupted
|
||||
* upgrade where v0_32_2 hasn't run could leave the cycle silently
|
||||
* misreporting "0 facts on people/alice" while legacy rows linger.
|
||||
*
|
||||
* The live-page requirement (#2484) is load-bearing: the inline facts
|
||||
* writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL`
|
||||
@@ -47,10 +41,6 @@
|
||||
* the phase jams forever (~16/day observed). Requiring a backing page
|
||||
* keeps genuine pre-v0.32.2 rows (whose entity page exists) gating
|
||||
* while excluding the inline-writer's permanent-unfenceable rows.
|
||||
*
|
||||
* Soft-expired rows don't count either (#2646): they're what
|
||||
* `forget_fact` produces, so excluding them lets operators drain the
|
||||
* backlog through the sanctioned removal path instead of raw SQL.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
@@ -98,24 +88,6 @@ function dedupeFactsByContentKey(facts: FenceExtractedFact[]): FenceExtractedFac
|
||||
* neither count as "stale" (which would force a wipe every cycle) nor
|
||||
* be compared against the fence's row set. Mirrors the
|
||||
* excludeSourcePrefixes filter deleteFactsForPage applies on the wipe.
|
||||
*
|
||||
* Also excludes soft-expired legacy rows (#2646: `row_num IS NULL AND
|
||||
* expired_at IS NOT NULL`) — rows that `forget_fact` expired via its
|
||||
* legacy DB-only path. They are not fence-owned (fence rows always
|
||||
* carry a row_num), so they must neither count as "stale" (forcing a
|
||||
* wipe every cycle) nor mask a fence row from insertion. Mirrors the
|
||||
* preserveExpiredLegacy filter deleteFactsForPage applies on the wipe.
|
||||
*
|
||||
* Deliberate consequence: if the fence still carries the same
|
||||
* (claim, source) as an expired legacy row, the reconcile inserts it
|
||||
* as a fresh ACTIVE fence-owned row. That is the fence-is-canonical
|
||||
* contract working as documented — legacy DB-only forgets "DO NOT
|
||||
* survive rebuild" (see forget.ts header); suppressing the insert
|
||||
* would instead create silent fence↔DB divergence, the exact failure
|
||||
* mode the empty-fence guard exists to prevent. To durably forget
|
||||
* such a claim, forget the fence-owned row (forget_fact now takes the
|
||||
* fence path, which strikes the row through in markdown). The expired
|
||||
* legacy row survives alongside as the record of the earlier forget.
|
||||
*/
|
||||
async function listExistingFactsForPage(
|
||||
engine: BrainEngine,
|
||||
@@ -128,7 +100,6 @@ async function listExistingFactsForPage(
|
||||
WHERE source_id = $1
|
||||
AND source_markdown_slug = $2
|
||||
AND COALESCE(source, '') NOT LIKE 'cli:%'
|
||||
AND NOT (row_num IS NULL AND expired_at IS NOT NULL)
|
||||
ORDER BY row_num ASC, id ASC`,
|
||||
[sourceId, slug],
|
||||
);
|
||||
@@ -202,7 +173,7 @@ export async function runExtractFacts(
|
||||
phantomsMorePending: false,
|
||||
};
|
||||
|
||||
// ── Empty-fence guard (Codex R2-#7; #2484; #2646) ──────────────
|
||||
// ── Empty-fence guard (Codex R2-#7; #2484) ─────────────────────
|
||||
// Pre-check: if any genuinely-backfillable legacy fact rows exist,
|
||||
// refuse to run the destructive reconciliation pass — the v0_32_2
|
||||
// orchestrator must fence them first.
|
||||
@@ -210,13 +181,12 @@ export async function runExtractFacts(
|
||||
// A row is a real backfill candidate only when `row_num IS NULL`
|
||||
// (never fenced) AND its `entity_slug` resolves to a LIVE page in
|
||||
// this source (the migration's Phase B only fences rows whose
|
||||
// entity_slug maps to a writable page) AND it is not soft-expired.
|
||||
// #2484: the original predicate was just `row_num IS NULL AND
|
||||
// entity_slug IS NOT NULL`, which ALSO matched
|
||||
// structurally-unfenceable hot-memory rows the inline writer keeps
|
||||
// producing post-migration: the legacy DB-only fallback
|
||||
// (backstop.ts) writes `entity_slug` (a resolved slug, e.g. a
|
||||
// slugify-floor or stub-guard-blocked unprefixed slug like
|
||||
// entity_slug maps to a writable page). #2484: the original
|
||||
// predicate was just `row_num IS NULL AND entity_slug IS NOT NULL`,
|
||||
// which ALSO matched structurally-unfenceable hot-memory rows the
|
||||
// inline writer keeps producing post-migration: the legacy DB-only
|
||||
// fallback (backstop.ts) writes `entity_slug` (a resolved slug, e.g.
|
||||
// a slugify-floor or stub-guard-blocked unprefixed slug like
|
||||
// `people-jane-doe`) with `row_num` NULL whenever the slug has no
|
||||
// fenceable page. Those rows can never satisfy the migration's exit
|
||||
// condition (no page to fence onto, and `apply-migrations` is a
|
||||
@@ -224,49 +194,27 @@ export async function runExtractFacts(
|
||||
// — ~16/day, mislabeled "v0.31 pending backfill." We now require a
|
||||
// live backing page, which both genuine pre-v0.32.2 rows (their
|
||||
// entity page exists) satisfy and inline-writer unfenceable rows do
|
||||
// not. #2646: soft-expired rows (`expired_at IS NOT NULL`) are also
|
||||
// excluded — `forget_fact`, the officially sanctioned removal path,
|
||||
// soft-expires legacy rows rather than deleting them, so counting
|
||||
// expired rows would leave the guard permanently stuck with no
|
||||
// supported way to drain the backlog.
|
||||
//
|
||||
// Source isolation (#3526): the count is scoped to THIS run's
|
||||
// sourceId. The pre-fix query counted brain-wide, so a single pending
|
||||
// legacy row in any mounted source jammed extract_facts for every
|
||||
// source — a cross-source leak of one source's migration state into
|
||||
// another's cycle (CLAUDE.md source-isolation invariant).
|
||||
// not.
|
||||
const legacy = await engine.executeRaw<{ n: string }>(
|
||||
`SELECT COUNT(*) AS n
|
||||
FROM facts f
|
||||
WHERE f.source_id = $1
|
||||
AND f.row_num IS NULL
|
||||
WHERE f.row_num IS NULL
|
||||
AND f.entity_slug IS NOT NULL
|
||||
AND f.expired_at IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM pages p
|
||||
WHERE p.source_id = f.source_id
|
||||
AND p.slug = f.entity_slug
|
||||
AND p.deleted_at IS NULL
|
||||
)`,
|
||||
[sourceId],
|
||||
);
|
||||
const legacyCount = parseInt(legacy[0]?.n ?? '0', 10);
|
||||
result.legacyRowsPending = legacyCount;
|
||||
if (legacyCount > 0) {
|
||||
result.guardTriggered = true;
|
||||
// Drain advice must actually work: a bare `apply-migrations --yes`
|
||||
// is a no-op once the v0.32.2 ledger entry says complete (the
|
||||
// runner classifies it as already-applied), so the sanctioned
|
||||
// re-run path is the explicit retry marker first. Phase B is
|
||||
// idempotent — it only touches `row_num IS NULL` rows and de-dupes
|
||||
// against the existing fence — so the re-run is safe. Individual
|
||||
// rows can instead be drained through `forget_fact` (soft-expired
|
||||
// rows stop counting).
|
||||
result.warnings.push(
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows in source "${sourceId}" ` +
|
||||
`(entity page present, not yet fenced) pending fence backfill. Re-run the v0.32.2 ` +
|
||||
`fence backfill: \`gbrain apply-migrations --force-retry 0.32.2\` then ` +
|
||||
`\`gbrain apply-migrations --yes\`. Or drain individual rows via \`forget_fact\`.`,
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` +
|
||||
`fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` +
|
||||
`v0_32_2 before this phase can safely reconcile fence → DB.`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
@@ -355,11 +303,6 @@ export async function runExtractFacts(
|
||||
result.warnings.push(
|
||||
...parsed.warnings.map(w => `${slug}: ${w}`),
|
||||
);
|
||||
// The parser deliberately skips malformed rows and returns any rows it
|
||||
// could still recover. That partial result is not authoritative: using
|
||||
// it for reconciliation would interpret skipped rows as deletions.
|
||||
// Preserve this page's existing index and continue with other pages.
|
||||
continue;
|
||||
}
|
||||
|
||||
if (parsed.facts.length > 0) result.pagesWithFacts += 1;
|
||||
@@ -391,12 +334,9 @@ export async function runExtractFacts(
|
||||
// partial-UNIQUE-index keyspace). #1928: `cli:`-origin facts
|
||||
// (conversation facts from extract-conversation-facts) are NOT
|
||||
// fence-owned — the page carries no `## Facts` fence to recreate
|
||||
// them — so they MUST survive this reconcile. #2646: soft-expired
|
||||
// legacy rows (forget_fact's record of the forget) likewise
|
||||
// survive via preserveExpiredLegacy.
|
||||
// them — so they MUST survive this reconcile.
|
||||
const deleted = await engine.deleteFactsForPage(slug, sourceId, {
|
||||
excludeSourcePrefixes: ['cli:'],
|
||||
preserveExpiredLegacy: true,
|
||||
});
|
||||
result.factsDeleted += deleted.deleted;
|
||||
}
|
||||
@@ -423,11 +363,10 @@ export async function runExtractFacts(
|
||||
if (hasStaleExisting || hasDuplicateExisting || hasRowNumDrift) {
|
||||
// Fall back to the legacy page-level reconcile when old DB rows must
|
||||
// be removed. Same delete scoping as above: legacy
|
||||
// NULL-source_markdown_slug rows, `cli:`-origin conversation
|
||||
// facts (#1928), and soft-expired legacy rows (#2646) survive.
|
||||
// NULL-source_markdown_slug rows and `cli:`-origin conversation
|
||||
// facts (#1928) survive.
|
||||
const deleted = await engine.deleteFactsForPage(slug, sourceId, {
|
||||
excludeSourcePrefixes: ['cli:'],
|
||||
preserveExpiredLegacy: true,
|
||||
});
|
||||
result.factsDeleted += deleted.deleted;
|
||||
toInsert = extracted;
|
||||
|
||||
@@ -219,7 +219,7 @@ export interface GradeTakesOpts extends BasePhaseOpts {
|
||||
/**
|
||||
* E2 ensemble judges. When useEnsemble=true and the single-model verdict
|
||||
* is borderline, all three judges are called in parallel via Promise.allSettled.
|
||||
* Defaults to [openai:gpt-5.2, anthropic:claude-sonnet-4-6, google:gemini-2.0-flash]
|
||||
* Defaults to [openai:gpt-4o, anthropic:claude-sonnet-4-6, google:gemini-1.5-pro]
|
||||
* via defaultJudge with model-string overrides. Tests inject deterministic
|
||||
* judges.
|
||||
*/
|
||||
|
||||
@@ -48,9 +48,8 @@ import { safeSplitIndex } from '../text-safe.ts';
|
||||
import { PAGE_SLUG_SEG } from '../cjk.ts';
|
||||
|
||||
// Slug grammar from validatePageSlug — shared via PAGE_SLUG_SEG (#738).
|
||||
// Used for the orchestrator-written summary index slug. `u` flag required
|
||||
// by PAGE_SLUG_SEG's \p{...} classes (#3417).
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'u');
|
||||
// Used for the orchestrator-written summary index slug.
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`);
|
||||
|
||||
// ── Model context budget (D1, D5, D7, D9) ─────────────────────────────
|
||||
|
||||
|
||||
@@ -112,7 +112,6 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'takes_weight_grid',
|
||||
'timeline_coverage',
|
||||
'unified_multimodal_coverage',
|
||||
'unverified_extractions',
|
||||
'voice_gate_health',
|
||||
]);
|
||||
|
||||
@@ -145,7 +144,6 @@ export const OPS_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'federation_health',
|
||||
'home_dir_in_worktree',
|
||||
'index_audit',
|
||||
'npm_squat',
|
||||
'oauth_confidential_client_health',
|
||||
'orphan_clones',
|
||||
'pgbouncer_prepare',
|
||||
|
||||
+2
-17
@@ -19,8 +19,7 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import { embedBatchWithBackoff, restampIfDemotedToTitleTier } from '../commands/embed.ts';
|
||||
import { wrapChunkTextsForStoredMode } from './embedding-context.ts';
|
||||
import { embedBatchWithBackoff } from '../commands/embed.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
|
||||
@@ -190,15 +189,8 @@ export async function embedStaleForSource(
|
||||
const keySourceId = stale[0]?.source_id ?? sourceId;
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes (mirrors
|
||||
// src/commands/embed.ts:embedAllStale).
|
||||
const pageRow = await observed(pacer, () =>
|
||||
engine.getPage(slug, { sourceId: keySourceId }),
|
||||
);
|
||||
const embeddings = await embedFn(
|
||||
wrapChunkTextsForStoredMode(pageRow, stale),
|
||||
stale.map((c) => c.chunk_text),
|
||||
{ abortSignal: signal },
|
||||
);
|
||||
const existing = await observed(pacer, () =>
|
||||
@@ -241,13 +233,6 @@ export async function embedStaleForSource(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest (mixed pages stay as-is).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
result.pagesProcessed += 1;
|
||||
} catch (e: unknown) {
|
||||
|
||||
@@ -186,41 +186,3 @@ export function modeRequiresHaiku(mode: CRMode): boolean {
|
||||
export function modeRequiresWrapper(mode: CRMode): boolean {
|
||||
return mode !== 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — build the embedding inputs for a re-embed of EXISTING chunk rows,
|
||||
* reproducing the wrapping convention the page's vectors were originally
|
||||
* built under (recorded in `pages.contextual_retrieval_mode`).
|
||||
*
|
||||
* Used by every plain re-embed path (`embed <slug>`, `embed --all`,
|
||||
* `embed --stale`, the embed-backfill Minion loop). Before this helper those
|
||||
* paths embedded raw `chunk_text`, so any re-embed — including the NORMAL
|
||||
* post-model-migration `embed --stale` — silently replaced context-wrapped
|
||||
* vectors with unwrapped ones, degrading retrieval with no signature change
|
||||
* to show for it.
|
||||
*
|
||||
* Convention rules (embed PRESERVES conventions; changing them is
|
||||
* sync/reindex's job):
|
||||
* - mode NULL/undefined/'none' → raw chunk_text (status quo).
|
||||
* - mode 'title' → title-only prefix (pure string concat).
|
||||
* - mode 'per_chunk_synopsis' → title-only prefix. Re-generating Haiku
|
||||
* synopses is a paid backfill concern; title-only is the service's own
|
||||
* documented fallback tier (D14). Callers that fully re-embed a page
|
||||
* this way should restamp the page to 'title' so the column stays
|
||||
* honest (see contextual-retrieval-service.ts:titleTierCorpusGeneration).
|
||||
* - `fenced_code` chunks are NEVER wrapped (D20-T4), same as sync.
|
||||
*/
|
||||
export function wrapChunkTextsForStoredMode(
|
||||
page:
|
||||
| { title?: string | null; contextual_retrieval_mode?: CRMode | null }
|
||||
| null
|
||||
| undefined,
|
||||
chunks: ReadonlyArray<{ chunk_text: string; chunk_source?: string | null }>,
|
||||
): string[] {
|
||||
const mode = page?.contextual_retrieval_mode;
|
||||
if (mode == null || !modeRequiresWrapper(mode)) {
|
||||
return chunks.map((c) => c.chunk_text);
|
||||
}
|
||||
const prefix = buildContextualPrefix(page?.title ?? '', null);
|
||||
return chunks.map((c) => wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source));
|
||||
}
|
||||
|
||||
@@ -5,15 +5,12 @@
|
||||
* cost-estimate prompt so users with large brains see a dollar figure
|
||||
* before the chunker-version sweep re-embeds.
|
||||
*
|
||||
* Prices in USD per 1M tokens. Every entry carries the official page it came
|
||||
* from plus the date it was last read against that page — re-verify alongside
|
||||
* the Anthropic-pricing refresh cycle; drift here produces estimates that
|
||||
* mislead operators. This table is for EMBEDDINGS only; chat/completion
|
||||
* pricing lives in `model-pricing.ts` (different unit) and must never be
|
||||
* mixed in here.
|
||||
* Prices in USD per 1M tokens. Numbers as of 2026-05-11. Verify alongside
|
||||
* the Anthropic-pricing refresh cycle; drift here produces estimates
|
||||
* that mislead operators.
|
||||
*
|
||||
* Codex outside-voice C3 fold: embedding providers with no entry below
|
||||
* (Hunyuan, Dashscope, etc.) return UNKNOWN_PROVIDER from `lookupPrice`
|
||||
* Codex outside-voice C3 fold: non-OpenAI embedding providers (Voyage,
|
||||
* Hunyuan, Dashscope, etc.) return UNKNOWN_PROVIDER from `lookupPrice`
|
||||
* so the cost-estimate prompt can fall back to a "estimate unavailable
|
||||
* for <provider>; press Ctrl-C in 10s to abort" message rather than
|
||||
* fabricate numbers.
|
||||
@@ -29,33 +26,25 @@ export interface EmbeddingPricing {
|
||||
* gateway model strings (e.g. 'openai:text-embedding-3-large').
|
||||
*/
|
||||
export const EMBEDDING_PRICING: Record<string, EmbeddingPricing> = {
|
||||
// OpenAI (https://developers.openai.com/api/docs/pricing, verified 2026-07-28)
|
||||
// OpenAI (https://openai.com/api/pricing/, verified 2026-05-11)
|
||||
'openai:text-embedding-3-large': { pricePerMTok: 0.13 },
|
||||
'openai:text-embedding-3-small': { pricePerMTok: 0.02 },
|
||||
// Legacy OpenAI ada (still common in older brains)
|
||||
'openai:text-embedding-ada-002': { pricePerMTok: 0.10 },
|
||||
// Voyage (https://docs.voyageai.com/docs/pricing, verified 2026-07-28)
|
||||
'voyage:voyage-4-large': { pricePerMTok: 0.12 },
|
||||
'voyage:voyage-4': { pricePerMTok: 0.06 },
|
||||
'voyage:voyage-4-lite': { pricePerMTok: 0.02 },
|
||||
// voyage-4-nano is deliberately absent: it's the open-weight variant (see
|
||||
// src/core/ai/recipes/voyage.ts) and Voyage's pricing page lists no hosted
|
||||
// rate for it. A 0 entry would under-estimate anyone paying for it via the
|
||||
// hosted API; no entry means lookupEmbeddingPrice returns `unknown` and the
|
||||
// caller prints "estimate unavailable" instead of a wrong number.
|
||||
// Legacy Voyage models (same page, "older models" section — no free tokens):
|
||||
// Voyage (https://www.voyageai.com/pricing)
|
||||
'voyage:voyage-3-large': { pricePerMTok: 0.18 },
|
||||
'voyage:voyage-3': { pricePerMTok: 0.06 },
|
||||
// ZeroEntropy (https://www.zeroentropy.dev/pricing, verified 2026-07-28)
|
||||
'voyage:voyage-4-large': { pricePerMTok: 0.18 },
|
||||
// ZeroEntropy (https://zeroentropy.dev/pricing — zembed-1)
|
||||
'zeroentropyai:zembed-1': { pricePerMTok: 0.05 },
|
||||
// ZeroEntropy reranker (docs/ai-providers/zeroentropy.md — $0.025/1M tokens).
|
||||
// Reused here (not a separate rerank table) because budget-tracker.ts's
|
||||
// rerank-kind lookup falls back to this same table for paid providers.
|
||||
'zeroentropyai:zerank-2': { pricePerMTok: 0.025 },
|
||||
// Mistral (https://mistral.ai/pricing/api/, verified 2026-07-28)
|
||||
// Mistral (https://mistral.ai/pricing/api/, verified 2026-07-19)
|
||||
'mistral:mistral-embed': { pricePerMTok: 0.10 },
|
||||
'mistral:mistral-embed-2312': { pricePerMTok: 0.10 },
|
||||
// Perplexity (https://docs.perplexity.ai/getting-started/pricing, verified 2026-07-28)
|
||||
// Perplexity (https://docs.perplexity.ai/getting-started/pricing, verified 2026-07-21)
|
||||
'perplexity:pplx-embed-v1-0.6b': { pricePerMTok: 0.004 },
|
||||
'perplexity:pplx-embed-v1-4b': { pricePerMTok: 0.03 },
|
||||
};
|
||||
|
||||
+2
-28
@@ -1341,16 +1341,6 @@ export interface BrainEngine {
|
||||
getContentFlagsByPageIds(
|
||||
pageIds: number[],
|
||||
): Promise<Map<number, { reason: string; detail: string }>>;
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160): for a list of page_ids, return
|
||||
* the subset that are unverified auto-extracted entity stubs (frontmatter
|
||||
* `provenance: 'auto-extracted'` + `status: 'unverified'`). Used by hybrid
|
||||
* search to stamp `SearchResult.unverified` pre-fusion so the fusion-level
|
||||
* compiled-truth boost skips them. Single SQL query, not N+1. Empty input
|
||||
* → empty set (no query). SQL predicate is the shared
|
||||
* `unverifiedExtractionFragment` (src/core/extraction-review.ts).
|
||||
*/
|
||||
getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>>;
|
||||
/**
|
||||
* v0.27.0: for a list of slugs, return their updated_at timestamps (or created_at fallback).
|
||||
* Used by hybrid search recency boost. Single SQL query, not N+1.
|
||||
@@ -1815,22 +1805,11 @@ export interface BrainEngine {
|
||||
* never recreate them (the page has no `## Facts` fence). Omitted ⇒ legacy
|
||||
* behavior (delete every fact on the page coordinate). NULL/empty `source`
|
||||
* rows are always deletable (fence default).
|
||||
*
|
||||
* #2646: `preserveExpiredLegacy` protects soft-expired legacy rows
|
||||
* (`row_num IS NULL AND expired_at IS NOT NULL`) — the record left by
|
||||
* `forget_fact`'s legacy DB-only path. Fence rows always carry a
|
||||
* `row_num`, so these rows are never fence-owned and a wipe would
|
||||
* destroy the forget record (the audit trail of the forget). Note what
|
||||
* this does NOT promise: it protects the record, not the forget itself —
|
||||
* if the fence still carries the same claim, fence canonicality
|
||||
* independently reinserts it as a fresh active row (legacy DB-only
|
||||
* forgets are documented as non-durable; see extract-facts.ts). Omitted
|
||||
* ⇒ legacy behavior.
|
||||
*/
|
||||
deleteFactsForPage(
|
||||
slug: string,
|
||||
source_id: string,
|
||||
opts?: { excludeSourcePrefixes?: string[]; preserveExpiredLegacy?: boolean },
|
||||
opts?: { excludeSourcePrefixes?: string[] },
|
||||
): Promise<{ deleted: number }>;
|
||||
|
||||
/**
|
||||
@@ -1951,13 +1930,8 @@ export interface BrainEngine {
|
||||
* preserved via stable page_id). `opts.sourceId` scopes the UPDATE — without
|
||||
* it, the bare `WHERE slug = old` matches every row across every source and
|
||||
* would either rename them all OR violate the (source_id, slug) UNIQUE.
|
||||
*
|
||||
* Returns the number of rows moved. 0 means the old slug had no row in the
|
||||
* scoped source — an UPDATE that matches nothing does NOT throw, so callers
|
||||
* that need to know whether the rename actually happened (the sync rename
|
||||
* path, #3056) must check the return value rather than rely on the catch.
|
||||
*/
|
||||
updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number>;
|
||||
updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void>;
|
||||
rewriteLinks(oldSlug: string, newSlug: string): Promise<void>;
|
||||
|
||||
/**
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { waitForCapacity } from './backoff.ts';
|
||||
import { quarantineMarkers } from './extraction-review.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -29,32 +28,9 @@ export interface EnrichmentRequest {
|
||||
tier?: 1 | 2 | 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trust options for the enrichment write path (issue #160).
|
||||
*
|
||||
* `trusted: true` — the input text comes from the machine owner via the
|
||||
* trusted local CLI (ctx.remote === false) AND the caller passed an explicit
|
||||
* opt-in flag. Stubs write direct as authoritative entity pages.
|
||||
*
|
||||
* Anything else (undefined, false, absent) is UNTRUSTED — fail-closed,
|
||||
* mirroring the OperationContext.remote invariant ("anything not strictly
|
||||
* false is remote"). Created stubs land in the quarantine lane: frontmatter
|
||||
* `provenance: 'auto-extracted'` + `status: 'unverified'`. They are excluded
|
||||
* from authoritative retrieval boosts and wait in the review queue
|
||||
* (`extraction_pending` / `extraction_review` ops) until the owner promotes
|
||||
* or rejects them.
|
||||
*/
|
||||
export interface EnrichmentTrustOptions {
|
||||
trusted?: boolean;
|
||||
/** Source to read/write in (multi-source brains). Omitted → engine default. */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface EnrichmentResult {
|
||||
slug: string;
|
||||
action: 'created' | 'updated' | 'skipped';
|
||||
/** True when the created stub landed in the quarantine lane (issue #160). */
|
||||
quarantined?: boolean;
|
||||
tier: 1 | 2 | 3;
|
||||
backlinkCreated: boolean;
|
||||
timelineAdded: boolean;
|
||||
@@ -96,15 +72,11 @@ export function entityPagePath(name: string, type: 'person' | 'company'): string
|
||||
export async function enrichEntity(
|
||||
engine: BrainEngine,
|
||||
request: EnrichmentRequest,
|
||||
opts?: EnrichmentTrustOptions,
|
||||
): Promise<EnrichmentResult> {
|
||||
const slug = slugifyEntity(request.entityName, request.entityType);
|
||||
// Fail-closed: only an explicit `trusted: true` writes authoritative pages.
|
||||
const trusted = opts?.trusted === true;
|
||||
const scope = opts?.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
|
||||
// 1. Count existing mentions for tier auto-escalation
|
||||
const { mentionCount, mentionSources } = await countMentions(engine, request.entityName, opts?.sourceId);
|
||||
const { mentionCount, mentionSources } = await countMentions(engine, request.entityName);
|
||||
|
||||
// 2. Determine tier (auto-escalate based on mentions)
|
||||
const suggestedTier = suggestTier(mentionCount, mentionSources, request.context);
|
||||
@@ -112,7 +84,7 @@ export async function enrichEntity(
|
||||
const tierEscalated = suggestedTier < (request.tier || 3); // lower tier number = higher importance
|
||||
|
||||
// 3. Check if entity page exists
|
||||
const existingPage = await engine.getPage(slug, scope);
|
||||
const existingPage = await engine.getPage(slug);
|
||||
let action: 'created' | 'updated' | 'skipped';
|
||||
|
||||
if (existingPage) {
|
||||
@@ -132,11 +104,8 @@ export async function enrichEntity(
|
||||
created: new Date().toISOString().split('T')[0],
|
||||
source: request.sourceSlug,
|
||||
tier,
|
||||
// issue #160 quarantine lane: stubs extracted from untrusted input
|
||||
// carry provenance + unverified markers until the owner reviews them.
|
||||
...(trusted ? {} : quarantineMarkers()),
|
||||
},
|
||||
}, scope);
|
||||
});
|
||||
action = 'created';
|
||||
}
|
||||
|
||||
@@ -147,7 +116,7 @@ export async function enrichEntity(
|
||||
date: new Date().toISOString().split('T')[0] ?? '',
|
||||
summary: `Referenced in [${request.sourceSlug}](${request.sourceSlug}) — ${request.context}`,
|
||||
source: request.sourceSlug,
|
||||
}, scope);
|
||||
});
|
||||
timelineAdded = true;
|
||||
} catch {
|
||||
// Timeline add failed (page might not support it)
|
||||
@@ -156,7 +125,7 @@ export async function enrichEntity(
|
||||
// 5. Add backlink from entity to source
|
||||
let backlinkCreated = false;
|
||||
try {
|
||||
await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`, undefined, undefined, undefined, undefined, opts?.sourceId ? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId } : undefined); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdown
|
||||
await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdown
|
||||
backlinkCreated = true;
|
||||
} catch {
|
||||
// Link might already exist
|
||||
@@ -165,7 +134,6 @@ export async function enrichEntity(
|
||||
return {
|
||||
slug,
|
||||
action,
|
||||
...(action === 'created' && !trusted ? { quarantined: true } : {}),
|
||||
tier,
|
||||
backlinkCreated,
|
||||
timelineAdded,
|
||||
@@ -184,14 +152,14 @@ export async function enrichEntity(
|
||||
export async function enrichEntities(
|
||||
engine: BrainEngine,
|
||||
requests: EnrichmentRequest[],
|
||||
config?: { throttle?: boolean; onProgress?: (done: number, total: number, name: string) => void } & EnrichmentTrustOptions,
|
||||
config?: { throttle?: boolean; onProgress?: (done: number, total: number, name: string) => void },
|
||||
): Promise<EnrichmentResult[]> {
|
||||
const results: EnrichmentResult[] = [];
|
||||
for (const req of requests) {
|
||||
if (config?.throttle !== false) {
|
||||
await waitForCapacity({ maxAttempts: 5 }); // shorter timeout for batch items
|
||||
}
|
||||
const result = await enrichEntity(engine, req, { trusted: config?.trusted, sourceId: config?.sourceId });
|
||||
const result = await enrichEntity(engine, req);
|
||||
results.push(result);
|
||||
config?.onProgress?.(results.length, requests.length, req.entityName);
|
||||
}
|
||||
@@ -207,11 +175,8 @@ export async function extractAndEnrich(
|
||||
engine: BrainEngine,
|
||||
text: string,
|
||||
sourceSlug: string,
|
||||
opts?: EnrichmentTrustOptions & { throttle?: boolean; maxEntities?: number },
|
||||
): Promise<EnrichmentResult[]> {
|
||||
// Bounded by default (#160 hardening): the greedy regex on a large paste
|
||||
// can produce thousands of hits; each enrichment is several DB round-trips.
|
||||
const entities = extractEntities(text).slice(0, opts?.maxEntities ?? 200);
|
||||
const entities = extractEntities(text);
|
||||
if (entities.length === 0) return [];
|
||||
|
||||
const requests: EnrichmentRequest[] = entities.map(e => ({
|
||||
@@ -221,7 +186,7 @@ export async function extractAndEnrich(
|
||||
sourceSlug,
|
||||
}));
|
||||
|
||||
return enrichEntities(engine, requests, { trusted: opts?.trusted, sourceId: opts?.sourceId, throttle: opts?.throttle });
|
||||
return enrichEntities(engine, requests);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -232,10 +197,9 @@ export async function extractAndEnrich(
|
||||
async function countMentions(
|
||||
engine: BrainEngine,
|
||||
entityName: string,
|
||||
sourceId?: string,
|
||||
): Promise<{ mentionCount: number; mentionSources: string[] }> {
|
||||
try {
|
||||
const results = await engine.searchKeyword(entityName, { limit: 100, ...(sourceId ? { sourceId } : {}) });
|
||||
const results = await engine.searchKeyword(entityName, { limit: 100 });
|
||||
// Derive sources from slug prefixes since SearchResult has no metadata.skill
|
||||
const sources = new Set<string>();
|
||||
for (const r of results) {
|
||||
|
||||
@@ -85,21 +85,11 @@ const RUN_ID_SHORT_LEN = 8;
|
||||
/**
|
||||
* Truncate a run id to the standard 8-char short form used in slug
|
||||
* paths. Idempotent — passing an already-short id returns it unchanged.
|
||||
* Non-hex / non-alphanumeric chars survive INSIDE the short form
|
||||
* (op-checkpoint ids may include dashes or other separators), but
|
||||
* boundary hyphens are trimmed (#3443): `slugifySegment()` strips
|
||||
* leading/trailing hyphens during repo sync, so a short form like
|
||||
* 'propose-' (from propose-<timestamp> run ids) made the DB receipt
|
||||
* slug and its Git-backed slug disagree — writing the receipt through
|
||||
* to the repo created a normalized sibling instead of materializing
|
||||
* the existing page. Invariant: slugifySegment(shortRunId(x)) ===
|
||||
* shortRunId(x) for slug-safe run ids.
|
||||
* Non-hex / non-alphanumeric chars survive (op-checkpoint ids may
|
||||
* include dashes or other separators).
|
||||
*/
|
||||
export function shortRunId(runId: string): string {
|
||||
// ponytail: truncation-based discrimination is only as good as the run id's
|
||||
// first 8 chars; families that need per-run uniqueness must front-load it.
|
||||
const short = runId.slice(0, RUN_ID_SHORT_LEN).replace(/^-+|-+$/g, '');
|
||||
return short || (runId ? 'run' : '');
|
||||
return runId.slice(0, RUN_ID_SHORT_LEN);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160).
|
||||
*
|
||||
* `extractAndEnrich` regex-extracts entity names from arbitrary ingested text
|
||||
* and creates `people/{slug}` / `companies/{slug}` stub pages. When the input
|
||||
* text comes from an untrusted channel (anything that is not the trusted local
|
||||
* CLI with an explicit opt-in), those stubs must NOT enter the brain as
|
||||
* authoritative entity pages. Instead they land in the quarantine lane:
|
||||
* ordinary pages carrying two frontmatter markers —
|
||||
*
|
||||
* provenance: 'auto-extracted' — HOW the page came to exist
|
||||
* status: 'unverified' — the owner has not reviewed it yet
|
||||
*
|
||||
* Consequences of the markers (each enforced at its own site):
|
||||
* - Search: unverified stubs are excluded from the compiled-truth authority
|
||||
* boost (they rank as ordinary content) and results carry
|
||||
* `unverified: true` so agents can label the provenance.
|
||||
* - Review: `extraction_pending` lists them; `extraction_review` promotes
|
||||
* (status → 'verified', provenance kept for audit) or rejects
|
||||
* (soft-delete) in batch. Promotion is local-owner-only.
|
||||
* - Doctor: counts unverified stubs older than N days as a review nudge.
|
||||
*
|
||||
* Fail-closed trust rule (mirrors OperationContext.remote): only an explicit
|
||||
* `trusted: true` writes direct; undefined/false/anything-else quarantines.
|
||||
*
|
||||
* Known scope (deliberate, documented — not gaps discovered later):
|
||||
* - CREATE-path only. The enrichment UPDATE path (timeline append + edge
|
||||
* onto an EXISTING page when a slug collides) is the separately-tracked
|
||||
* slug-collision finding referenced in issue #160; this lane does not
|
||||
* gate it.
|
||||
* - The markers are ordinary frontmatter keys, not put_page-strip-listed
|
||||
* (#1699). A caller holding generic remote put_page write scope can
|
||||
* rewrite a stub without them — but that caller can author an unmarked
|
||||
* people/ page directly anyway, so stripping here adds no privilege.
|
||||
* The promotion OP surface (extraction_review) is what stays owner-only.
|
||||
*
|
||||
* Sibling of `src/core/quarantine.ts` / `src/core/embed-skip.ts` — same
|
||||
* marker-as-frontmatter-JSONB pattern, same "SQL fragment lives next to the
|
||||
* marker key so they can never drift" rule. No schema migration needed.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Marker keys + values (stable contract)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const EXTRACTION_PROVENANCE_KEY = 'provenance';
|
||||
export const EXTRACTION_STATUS_KEY = 'status';
|
||||
|
||||
export const PROVENANCE_AUTO_EXTRACTED = 'auto-extracted';
|
||||
export const STATUS_UNVERIFIED = 'unverified';
|
||||
export const STATUS_VERIFIED = 'verified';
|
||||
|
||||
/** Frontmatter markers to spread onto a quarantined stub at create time. */
|
||||
export function quarantineMarkers(): Record<string, string> {
|
||||
return {
|
||||
[EXTRACTION_PROVENANCE_KEY]: PROVENANCE_AUTO_EXTRACTED,
|
||||
[EXTRACTION_STATUS_KEY]: STATUS_UNVERIFIED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* JS-side predicate: true only when BOTH markers match. Requiring the pair
|
||||
* means user pages that happen to carry their own `status` or `provenance`
|
||||
* frontmatter are never captured by the review lane.
|
||||
*/
|
||||
export function isUnverifiedExtraction(
|
||||
frontmatter: Record<string, unknown> | null | undefined,
|
||||
): boolean {
|
||||
if (!frontmatter) return false;
|
||||
return (
|
||||
frontmatter[EXTRACTION_PROVENANCE_KEY] === PROVENANCE_AUTO_EXTRACTED &&
|
||||
frontmatter[EXTRACTION_STATUS_KEY] === STATUS_UNVERIFIED
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment matching unverified auto-extracted stubs, parameterized on the
|
||||
* page-table alias. Single source of truth for every SQL-side consumer
|
||||
* (extraction_pending list, doctor count) so the filter and the marker keys
|
||||
* can never drift. `pageAlias` is engine-supplied (never user input).
|
||||
* JSONB `->>` works identically on Postgres and PGLite (PostgreSQL-in-WASM).
|
||||
*/
|
||||
export function unverifiedExtractionFragment(pageAlias: string): string {
|
||||
return (
|
||||
`(COALESCE(${pageAlias}.frontmatter, '{}'::jsonb) ->> '${EXTRACTION_PROVENANCE_KEY}') = '${PROVENANCE_AUTO_EXTRACTED}'` +
|
||||
` AND (COALESCE(${pageAlias}.frontmatter, '{}'::jsonb) ->> '${EXTRACTION_STATUS_KEY}') = '${STATUS_UNVERIFIED}'`
|
||||
);
|
||||
}
|
||||
+13
-55
@@ -96,71 +96,29 @@ export interface GitFreshnessOpts {
|
||||
}
|
||||
|
||||
/**
|
||||
* Three-state git probe verdict for a federated source clone.
|
||||
*
|
||||
* - `'unchanged'`: HEAD matches `last_commit` (and, when requested, the
|
||||
* working tree is clean). Sync has nothing to do.
|
||||
* - `'changed'`: the clone is readable but HEAD moved, the tree is
|
||||
* dirty, or the DB never recorded a `last_commit` —
|
||||
* sync genuinely has (or may have) work.
|
||||
* - `'unavailable'`: the HEAD probe itself could not run — the clone
|
||||
* directory is missing, not a git repo, or git errored.
|
||||
* On stateless deploys (containers on EB / K8s / Fly,
|
||||
* where `local_path` dies with the filesystem and is
|
||||
* lazily re-materialized by the next per-source sync)
|
||||
* this is a NORMAL steady state for quiet sources, not
|
||||
* evidence of pending work. Callers can fall back to a
|
||||
* DB-only freshness signal instead of wall-clock age.
|
||||
*/
|
||||
export type SourceGitState = 'unchanged' | 'changed' | 'unavailable';
|
||||
|
||||
/**
|
||||
* Probe a source clone and classify it (see `SourceGitState`).
|
||||
* Returns true iff `localPath` is a git repo whose current HEAD matches
|
||||
* `lastCommit`, AND (when `requireCleanWorkingTree`) the working tree
|
||||
* is clean.
|
||||
*
|
||||
* This is NOT a full mirror of `gbrain sync`'s "do work?" predicate.
|
||||
* Chunker-version match is computed by the caller because it depends on
|
||||
* engine state (`sources.chunker_version` vs `CURRENT_CHUNKER_VERSION`).
|
||||
* See `src/commands/doctor.ts:checkSyncFreshness` for the AND
|
||||
* combination at the call site.
|
||||
*
|
||||
* NULL-input guard stays first: a NULL `last_commit` (legacy row) returns
|
||||
* `'changed'` WITHOUT running the head probe — same short-circuit contract
|
||||
* `isSourceUnchangedSinceSync` always had (pinned by doctor.test.ts case 4).
|
||||
*/
|
||||
export function probeSourceGitState(
|
||||
localPath: string | null | undefined,
|
||||
lastCommit: string | null | undefined,
|
||||
opts?: GitFreshnessOpts,
|
||||
): SourceGitState {
|
||||
if (!localPath || !lastCommit) return 'changed';
|
||||
const head = _headProbe(localPath);
|
||||
if (head === null) return 'unavailable';
|
||||
if (head !== lastCommit) return 'changed';
|
||||
if (opts?.requireCleanWorkingTree) {
|
||||
const ignoreUntracked = opts.requireCleanWorkingTree === 'ignore-untracked';
|
||||
const isClean = _cleanProbe(localPath, ignoreUntracked);
|
||||
// null (probe error) AND false (known dirty) both fail the gate. A clean
|
||||
// probe error with a READABLE head is not classified 'unavailable' —
|
||||
// fail toward "may have work" so the gate can only relax, never mask.
|
||||
if (isClean !== true) return 'changed';
|
||||
}
|
||||
return 'unchanged';
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true iff `localPath` is a git repo whose current HEAD matches
|
||||
* `lastCommit`, AND (when `requireCleanWorkingTree`) the working tree
|
||||
* is clean.
|
||||
*
|
||||
* Boolean façade over `probeSourceGitState` — `'unavailable'` and
|
||||
* `'changed'` both collapse to `false`, preserving the v0.41.27.0
|
||||
* fail-open contract for callers that only care about the short-circuit
|
||||
* (`src/core/source-health.ts`).
|
||||
*/
|
||||
export function isSourceUnchangedSinceSync(
|
||||
localPath: string | null | undefined,
|
||||
lastCommit: string | null | undefined,
|
||||
opts?: GitFreshnessOpts,
|
||||
): boolean {
|
||||
return probeSourceGitState(localPath, lastCommit, opts) === 'unchanged';
|
||||
if (!localPath || !lastCommit) return false;
|
||||
const head = _headProbe(localPath);
|
||||
if (head === null || head !== lastCommit) return false;
|
||||
if (opts?.requireCleanWorkingTree) {
|
||||
const ignoreUntracked = opts.requireCleanWorkingTree === 'ignore-untracked';
|
||||
const isClean = _cleanProbe(localPath, ignoreUntracked);
|
||||
// null (probe error) AND false (known dirty) both fail the gate.
|
||||
if (isClean !== true) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
+5
-51
@@ -39,7 +39,6 @@ import { normalizeAliasList } from './search/alias-normalize.ts';
|
||||
import { isUndefinedTableError, warnOncePerProcess, validateSlug } from './utils.ts';
|
||||
import { computeCorpusGeneration } from './contextual-retrieval-service.ts';
|
||||
import { runGuardrails } from './guardrails.ts';
|
||||
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END, parseFactsFence } from './facts-fence.ts';
|
||||
|
||||
/**
|
||||
* v0.20.0 Cathedral II Layer 8 D2 — markdown fence extraction helper.
|
||||
@@ -105,27 +104,6 @@ function fenceTagToPseudoPath(lang: string | undefined): string | null {
|
||||
*/
|
||||
const MAX_FENCES_PER_PAGE = Number.parseInt(process.env.GBRAIN_MAX_FENCES_PER_PAGE || '100', 10);
|
||||
|
||||
function extractFactsFenceBlock(body: string): string | null {
|
||||
const beginIdx = body.indexOf(FACTS_FENCE_BEGIN);
|
||||
if (beginIdx === -1) return null;
|
||||
const endIdx = body.indexOf(FACTS_FENCE_END, beginIdx + FACTS_FENCE_BEGIN.length);
|
||||
if (endIdx === -1) return null;
|
||||
return body.slice(beginIdx, endIdx + FACTS_FENCE_END.length);
|
||||
}
|
||||
|
||||
function replaceOrAppendFactsFence(body: string, fenceBlock: string): string {
|
||||
const beginIdx = body.indexOf(FACTS_FENCE_BEGIN);
|
||||
if (beginIdx !== -1) {
|
||||
const endIdx = body.indexOf(FACTS_FENCE_END, beginIdx + FACTS_FENCE_BEGIN.length);
|
||||
if (endIdx !== -1) {
|
||||
return body.slice(0, beginIdx) + fenceBlock + body.slice(endIdx + FACTS_FENCE_END.length);
|
||||
}
|
||||
}
|
||||
|
||||
const sep = body.endsWith('\n') ? '\n' : '\n\n';
|
||||
return `${body}${sep}## Facts\n\n${fenceBlock}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the marked lexer output and extract recognizable code fences.
|
||||
* Returns one ChunkInput per fence whose language tag maps to a grammar
|
||||
@@ -570,26 +548,6 @@ export async function importFromContent(
|
||||
// hash-match skip) and (b) the hash short-circuit below reuses this row.
|
||||
const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined);
|
||||
|
||||
// #2044: remote get_page intentionally strips private facts rows. A
|
||||
// documented get_page -> edit -> put_page round-trip can therefore arrive
|
||||
// with an empty/missing Facts fence even though the existing page still has
|
||||
// canonical fence rows. Preserve the old fence in that narrow case so the
|
||||
// system-of-record markdown is not truncated by the privacy boundary.
|
||||
if (opts.remote === true && existing?.compiled_truth) {
|
||||
const incomingFacts = parseFactsFence(parsed.compiled_truth);
|
||||
const existingFacts = parseFactsFence(existing.compiled_truth);
|
||||
const existingFenceBlock = extractFactsFenceBlock(existing.compiled_truth);
|
||||
if (
|
||||
incomingFacts.facts.length === 0 &&
|
||||
incomingFacts.warnings.length === 0 &&
|
||||
existingFacts.warnings.length === 0 &&
|
||||
existingFacts.facts.length > 0 &&
|
||||
existingFenceBlock
|
||||
) {
|
||||
parsed.compiled_truth = replaceOrAppendFactsFence(parsed.compiled_truth, existingFenceBlock);
|
||||
}
|
||||
}
|
||||
|
||||
// #1035: absence of an explicit frontmatter `type:` on an EXISTING page
|
||||
// means "preserve the stored type", not "re-infer". Pre-fix, a round-trip
|
||||
// put (get_page → edit body → put_page without `type:`) silently regressed
|
||||
@@ -1134,8 +1092,8 @@ export async function importFromFile(
|
||||
chunks: 0,
|
||||
error:
|
||||
`Filename "${relativePath}" produces no usable slug. ` +
|
||||
`Add a "slug:" to the frontmatter, or rename the file to include ` +
|
||||
`at least one letter or number (any script).`,
|
||||
`Add a "slug:" to the frontmatter, or rename the file to use ` +
|
||||
`ASCII / Chinese / Japanese / Korean characters.`,
|
||||
};
|
||||
}
|
||||
} else if (parsed.slug !== expectedSlug) {
|
||||
@@ -1203,10 +1161,6 @@ export async function importCodeFile(
|
||||
const title = `${relativePath} (${lang})`;
|
||||
const sourceId = opts.sourceId;
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
// PostgreSQL text columns reject U+0000 even though source files may
|
||||
// legitimately contain it inside string/regex fixtures. Preserve a visible,
|
||||
// searchable representation instead of dropping the entire code page.
|
||||
const storageContent = content.replaceAll('\0', '\\0');
|
||||
|
||||
const byteLength = Buffer.byteLength(content, 'utf-8');
|
||||
if (byteLength > MAX_FILE_SIZE) {
|
||||
@@ -1248,7 +1202,7 @@ export async function importCodeFile(
|
||||
// from the chunker (nested methods carry ['ClassName'] etc.) so the
|
||||
// chunk-grain FTS trigger picks up scope for ranking and downstream
|
||||
// Layer 5 edge resolution can use scope-qualified identity.
|
||||
const { chunks: codeChunks, edges: extractedEdges } = await chunkCodeTextFull(storageContent, relativePath);
|
||||
const { chunks: codeChunks, edges: extractedEdges } = await chunkCodeTextFull(content, relativePath);
|
||||
const chunks: ChunkInput[] = codeChunks.map((c, i) => ({
|
||||
chunk_index: i,
|
||||
chunk_text: c.text,
|
||||
@@ -1316,7 +1270,7 @@ export async function importCodeFile(
|
||||
type: 'code' as string,
|
||||
page_kind: 'code',
|
||||
title,
|
||||
compiled_truth: storageContent,
|
||||
compiled_truth: content,
|
||||
timeline: '',
|
||||
frontmatter: { language: lang, file: relativePath },
|
||||
content_hash: hash,
|
||||
@@ -1388,7 +1342,7 @@ export async function importCodeFile(
|
||||
|
||||
const edgeInputs: import('./types.ts').CodeEdgeInput[] = [];
|
||||
for (const e of extractedEdges) {
|
||||
const idx = findChunkForOffset(e.callSiteByteOffset, storageContent, rangeList);
|
||||
const idx = findChunkForOffset(e.callSiteByteOffset, content, rangeList);
|
||||
if (idx == null) continue;
|
||||
const from = rangeList[idx]!;
|
||||
if (!from.id || !from.symbol_name_qualified) continue;
|
||||
|
||||
+15
-80
@@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { PageType, EffectiveDateSource } from './types.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import { ensureWellFormed } from './text-safe.ts';
|
||||
|
||||
/**
|
||||
@@ -28,11 +28,10 @@ import { ensureWellFormed } from './text-safe.ts';
|
||||
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
|
||||
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
|
||||
*/
|
||||
// 2026-07-30: bumped for the #3466 inferTypeByDir fix — unevidenced
|
||||
// people/ -> companies/ adjacency now infers 'mentions' instead of
|
||||
// 'works_at'; the bump re-flags stamped pages so the next --stale sweep
|
||||
// re-extracts them under the corrected inference.
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-30T00:00:00Z';
|
||||
// 2026-07-10: bumped for the #2576 --stale nullResolver fix — sweeps before it
|
||||
// stamped pages with their bare wikilinks silently dropped; the bump re-flags
|
||||
// them so the fixed sweep re-extracts.
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-10T00:00:00Z';
|
||||
|
||||
// ─── Entity references ──────────────────────────────────────────
|
||||
|
||||
@@ -672,17 +671,6 @@ const FOUNDED_RE = /\b(?:founded|co-?founded|started the company|incorporated|fo
|
||||
// "security advisor to|at", "product advisor to|at", "industry advisor".
|
||||
const ADVISES_RE = /\b(?:advises|advised|advisor (?:to|at|for|of)|advisory (?:board|role|position|capacity|engagement|partnership|contract|relationship|work)|board advisor|on .{0,20} advisory board|joined .{0,20} advisory board|in an? advisory (?:capacity|role|position)|as an? (?:advisor|security advisor|technical advisor|strategic advisor|industry advisor|product advisor|board advisor|senior advisor)|(?:strategic|technical|security|product|industry|senior|board) advisor (?:to|at|for|of)|consults for|consulting role (?:at|with))\b/i;
|
||||
|
||||
// Chinese link type patterns for CJK entity mentions.
|
||||
// NOTE: These patterns are Chinese-only (zh). Japanese and Korean link
|
||||
// type extraction is not yet implemented. Entity NAME extraction in
|
||||
// by-mention.ts covers all three scripts (CJK = Chinese/Japanese/Korean)
|
||||
// via Unicode-aware tokenization.
|
||||
const ZH_FOUNDED_RE = /(?:创立|创办|成立|创建|建立|开创|发起)(?:了|的)/;
|
||||
const ZH_INVESTED_RE = /(?:投资|入股|融资|注资|参股)(?:了|的|了?于)/;
|
||||
const ZH_ADVISES_RE = /(?:顾问|咨询|指导)(?:了|的)?/;
|
||||
const ZH_WORKS_AT_RE = /(?:任职|就职|担任|供职|在.{0,10}(?:工作|上班|负责))(?:于|在|的)?/;
|
||||
const ZH_CITED_RE = /(?:引用|援引|提到|提及|转述|摘录)(?:了|的|自)?/;
|
||||
|
||||
// Page-role detection: if the source page describes a partner/investor at
|
||||
// page level, that's a strong prior for outbound company refs being
|
||||
// invested_in even when per-edge context lacks explicit investment verbs.
|
||||
@@ -736,12 +724,6 @@ export function inferLinkType(pageType: PageType, context: string, globalContext
|
||||
if (INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ADVISES_RE.test(context)) return 'advises';
|
||||
if (WORKS_AT_RE.test(context)) return 'works_at';
|
||||
// Chinese link type patterns
|
||||
if (ZH_FOUNDED_RE.test(context)) return 'founded';
|
||||
if (ZH_INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ZH_ADVISES_RE.test(context)) return 'advises';
|
||||
if (ZH_WORKS_AT_RE.test(context)) return 'works_at';
|
||||
if (ZH_CITED_RE.test(context)) return 'cited';
|
||||
// Page-role prior: only fires for person -> company links. Concept pages
|
||||
// about VC topics naturally contain "venture capital" in their text, but
|
||||
// their company refs are mentions, not investments. Partner pages mentioning
|
||||
@@ -1192,10 +1174,6 @@ export interface TimelineCandidate {
|
||||
// Match: `- **YYYY-MM-DD** | summary` or `- **YYYY-MM-DD** -- summary`
|
||||
// or `- **YYYY-MM-DD** - summary` or just `**YYYY-MM-DD** | summary`.
|
||||
const TIMELINE_LINE_RE = /^\s*-?\s*\*\*(\d{4}-\d{2}-\d{2})\*\*\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
// Chinese date lines: `- 2020年1月2日 | summary` (bold optional). Requires the
|
||||
// 年/月 markers so plain ASCII `- 2020-01-02 - text` does NOT match — non-bold
|
||||
// ASCII dates were never timeline entries and must stay that way.
|
||||
const TIMELINE_LINE_RE_CN = /^\s*-?\s*(?:\*\*)?(\d{4})年(\d{1,2})月(\d{1,2})日?(?:\*\*)?\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
|
||||
/**
|
||||
* Parse timeline entries from content. Looks at:
|
||||
@@ -1212,21 +1190,18 @@ export function parseTimelineEntries(content: string): TimelineCandidate[] {
|
||||
|
||||
let i = 0;
|
||||
while (i < lines.length) {
|
||||
// Try English format first, then Chinese
|
||||
const m = TIMELINE_LINE_RE.exec(lines[i]);
|
||||
let date: string;
|
||||
let summary: string;
|
||||
if (m) {
|
||||
date = m[1];
|
||||
summary = m[2].trim();
|
||||
} else {
|
||||
const cm = TIMELINE_LINE_RE_CN.exec(lines[i]);
|
||||
if (!cm) { i++; continue; }
|
||||
// Normalize Chinese date to YYYY-MM-DD
|
||||
date = `${cm[1]}-${cm[2].padStart(2, '0')}-${cm[3].padStart(2, '0')}`;
|
||||
summary = cm[4].trim();
|
||||
if (!m) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (!isValidDate(date) || summary.length === 0) { i++; continue; }
|
||||
const date = m[1];
|
||||
const summary = m[2].trim();
|
||||
if (!isValidDate(date) || summary.length === 0) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Collect optional detail lines (indented, until next date or heading).
|
||||
const detailLines: string[] = [];
|
||||
let j = i + 1;
|
||||
@@ -1291,46 +1266,6 @@ function isValidDate(s: string): boolean {
|
||||
return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d;
|
||||
}
|
||||
|
||||
/** Input for {@link deriveTimelineAnchor}: a page's identity + its computed content date. */
|
||||
export interface TimelineAnchorInput {
|
||||
slug: string;
|
||||
title?: string | null;
|
||||
effectiveDate?: Date | string | null;
|
||||
effectiveDateSource?: EffectiveDateSource | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Anchor a single timeline entry from a page's computed content date, for pages
|
||||
* whose body carries no parseable timeline line.
|
||||
*
|
||||
* Comms- and calendar-dominated brains keep the date in frontmatter or the
|
||||
* filename (slug `2026-04-24-...`), not in the prose, so `parseTimelineEntries`
|
||||
* returns nothing and the page-level `timeline` table stays empty even though
|
||||
* the page is firmly dated — leaving `get_timeline` and the brain-score
|
||||
* `timeline_coverage` component blind to it. This recovers that signal from the
|
||||
* already-computed `effective_date` (no re-parsing). (It does NOT feed the
|
||||
* facts-based `find_trajectory`, which reads the `facts` table by entity_slug.)
|
||||
*
|
||||
* Fires ONLY for a trustworthy content date — frontmatter (`event_date` / `date`
|
||||
* / `published`) or the `filename` date — never the `fallback` source, which is
|
||||
* `updated_at` (link-churn noise, not when the thing happened). Returns null
|
||||
* when no trustworthy date is available. Callers MUST apply this only when body
|
||||
* parsing yields zero entries, so it can never shadow a real in-body timeline.
|
||||
*/
|
||||
export function deriveTimelineAnchor(input: TimelineAnchorInput): TimelineCandidate | null {
|
||||
const { slug, title, effectiveDate, effectiveDateSource } = input;
|
||||
if (!effectiveDate) return null;
|
||||
// 'fallback' === updated_at; the rest ('event_date'|'date'|'published'|'filename')
|
||||
// are real content dates. null/undefined source is not trustworthy either.
|
||||
if (effectiveDateSource == null || effectiveDateSource === 'fallback') return null;
|
||||
const dt = typeof effectiveDate === 'string' ? new Date(effectiveDate) : effectiveDate;
|
||||
if (!(dt instanceof Date) || Number.isNaN(dt.getTime())) return null;
|
||||
const iso = dt.toISOString().slice(0, 10);
|
||||
if (!isValidDate(iso)) return null;
|
||||
const summary = (title ?? '').trim() || slug.split('/').pop() || slug;
|
||||
return { date: iso, summary, detail: '' };
|
||||
}
|
||||
|
||||
// ─── Auto-link config ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
+125
-17
@@ -2,13 +2,6 @@ import type { BrainEngine } from './engine.ts';
|
||||
import { slugifyPath } from './sync.ts';
|
||||
import { getFtsLanguage } from './fts-language.ts';
|
||||
import { hnswMaxDimsForType } from './vector-index.ts';
|
||||
// runMigrations executes while an initialized engine is live. Keep its helper
|
||||
// modules in the static graph rather than importing them from async handlers.
|
||||
import {
|
||||
isStatementTimeoutError,
|
||||
isRetryableConnError,
|
||||
} from './retry-matcher.ts';
|
||||
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
|
||||
|
||||
/**
|
||||
* Schema migrations — run automatically on initSchema().
|
||||
@@ -546,7 +539,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
sql: '',
|
||||
handler: async (engine) => {
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 14, 'idx_pages_updated_at_desc');
|
||||
await engine.runMigration(
|
||||
14,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'idx_pages_updated_at_desc' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS idx_pages_updated_at_desc';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
14,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pages_updated_at_desc
|
||||
@@ -1652,7 +1656,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
// 3. Partial index for the autopilot purge sweep. Postgres CONCURRENTLY
|
||||
// avoids the SHARE lock on `pages`; PGLite has no concurrent writers.
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 34, 'pages_deleted_at_purge_idx');
|
||||
// Pre-drop any invalid index from a prior CONCURRENTLY failure (matches v14 pattern).
|
||||
await engine.runMigration(34, `
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_deleted_at_purge_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_deleted_at_purge_idx';
|
||||
END IF;
|
||||
END $$;
|
||||
`);
|
||||
await engine.runMigration(34, `
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_deleted_at_purge_idx
|
||||
ON pages (deleted_at) WHERE deleted_at IS NOT NULL;
|
||||
@@ -1989,7 +2004,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
|
||||
// 2. Expression index for since/until date-range filters.
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 38, 'pages_coalesce_date_idx');
|
||||
// Pre-drop any invalid index from a prior CONCURRENTLY failure.
|
||||
await engine.runMigration(38, `
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_coalesce_date_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_coalesce_date_idx';
|
||||
END IF;
|
||||
END $$;
|
||||
`);
|
||||
await engine.runMigration(38, `
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_coalesce_date_idx
|
||||
ON pages ((COALESCE(effective_date, updated_at)));
|
||||
@@ -3551,7 +3577,19 @@ export const MIGRATIONS: Migration[] = [
|
||||
sql: '',
|
||||
handler: async (engine) => {
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 71, 'takes_resolved_at_idx');
|
||||
// Pre-drop invalid remnant from a failed CONCURRENTLY attempt.
|
||||
await engine.runMigration(
|
||||
71,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'takes_resolved_at_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS takes_resolved_at_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
71,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS takes_resolved_at_idx
|
||||
@@ -4211,7 +4249,20 @@ export const MIGRATIONS: Migration[] = [
|
||||
await engine.runMigration(91, columnsAndTrigger);
|
||||
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 91, 'pages_generation_idx');
|
||||
// Pre-drop any invalid index from a prior CONCURRENTLY failure
|
||||
// (matches v14 pattern).
|
||||
await engine.runMigration(
|
||||
91,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_generation_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_generation_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
91,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_generation_idx ON pages (generation);`
|
||||
@@ -4465,7 +4516,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
sql: '',
|
||||
handler: async (engine) => {
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 96, 'idx_facts_extract_conversation_session');
|
||||
await engine.runMigration(
|
||||
96,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'idx_facts_extract_conversation_session' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS idx_facts_extract_conversation_session';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
96,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_facts_extract_conversation_session
|
||||
@@ -4507,7 +4569,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
transaction: false,
|
||||
handler: async (engine) => {
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 97, 'pages_dedup_idx');
|
||||
await engine.runMigration(
|
||||
97,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_dedup_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_dedup_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
97,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_dedup_idx
|
||||
@@ -4675,7 +4748,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
);
|
||||
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 103, 'content_chunks_stale_idx');
|
||||
await engine.runMigration(
|
||||
103,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'content_chunks_stale_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS content_chunks_stale_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
103,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS content_chunks_stale_idx
|
||||
@@ -4709,7 +4793,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
sql: '',
|
||||
handler: async (engine) => {
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 104, 'pages_atom_source_hash_idx');
|
||||
await engine.runMigration(
|
||||
104,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_atom_source_hash_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_atom_source_hash_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
104,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_atom_source_hash_idx
|
||||
@@ -5010,7 +5105,18 @@ export const MIGRATIONS: Migration[] = [
|
||||
`ALTER TABLE pages ADD COLUMN IF NOT EXISTS links_extracted_at TIMESTAMPTZ;`
|
||||
);
|
||||
if (engine.kind === 'postgres') {
|
||||
await dropInvalidConcurrentIndex(engine, 112, 'pages_links_extracted_at_idx');
|
||||
await engine.runMigration(
|
||||
112,
|
||||
`DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_links_extracted_at_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_links_extracted_at_idx';
|
||||
END IF;
|
||||
END $$;`
|
||||
);
|
||||
await engine.runMigration(
|
||||
112,
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_links_extracted_at_idx
|
||||
@@ -5695,6 +5801,7 @@ async function runMigrationSQLWithRetry(
|
||||
m: Migration,
|
||||
sql: string,
|
||||
): Promise<void> {
|
||||
const { isStatementTimeoutError, isRetryableConnError } = await import('./retry-matcher.ts');
|
||||
// GBRAIN_MIGRATE_BACKOFF_MS lets tests skip the 5s/15s/45s backoff. In
|
||||
// production the env var is unset and the default cadence applies.
|
||||
const fastBackoff = process.env.GBRAIN_MIGRATE_BACKOFF_MS;
|
||||
@@ -5964,6 +6071,7 @@ export async function runMigrations(engine: BrainEngine): Promise<{ applied: num
|
||||
// reach the loop below). Best-effort + idempotent: a no-op on a healthy
|
||||
// index; `doctor` surfaces it independently if this ever fails.
|
||||
try {
|
||||
const { repairTimelineDedupIndex } = await import('./timeline-dedup-repair.ts');
|
||||
const r = await repairTimelineDedupIndex(engine);
|
||||
if (r.repaired) {
|
||||
console.error(
|
||||
|
||||
@@ -43,11 +43,6 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
|
||||
// few writes. Generous 10-min budget (vs the tight null-default) covers a
|
||||
// slow gateway without the 30-min loop budget.
|
||||
chronicle_extract: TEN_MIN_MS,
|
||||
// #3207 — same shape as chronicle_extract: one page = one LLM extraction
|
||||
// call + a few writes. Was missing from this map, so it inherited the tight
|
||||
// null-default and got dead-lettered mid-generation on slow chat providers
|
||||
// (facts silently lost) — exactly the failure this file exists to prevent.
|
||||
'facts-absorb': TEN_MIN_MS,
|
||||
// Per-page contextual reindex jobs process chunks sequentially with one
|
||||
// rate-leased LLM synopsis call per chunk; large transcript pages need more
|
||||
// than the standard 30-min long-job budget.
|
||||
|
||||
@@ -534,21 +534,10 @@ export class MinionQueue {
|
||||
}
|
||||
|
||||
/** Prune old jobs in terminal statuses. Returns count of deleted rows. */
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[]; dryRun?: boolean }): Promise<number> {
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[] }): Promise<number> {
|
||||
const statuses = opts?.status ?? ['completed', 'dead', 'cancelled'];
|
||||
const olderThan = opts?.olderThan ?? new Date(Date.now() - 30 * 86400000);
|
||||
|
||||
// #2712: dryRun counts the would-be-pruned rows without deleting.
|
||||
// Silent-ignoring a safety flag on a delete path is data loss.
|
||||
if (opts?.dryRun) {
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text as count FROM minion_jobs
|
||||
WHERE status = ANY($1) AND updated_at < $2`,
|
||||
[statuses, olderThan.toISOString()]
|
||||
);
|
||||
return parseInt(rows[0]?.count ?? '0', 10);
|
||||
}
|
||||
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`WITH pruned AS (
|
||||
DELETE FROM minion_jobs
|
||||
|
||||
@@ -84,9 +84,6 @@ export const CANONICAL_PRICING: Record<string, ModelPricing> = {
|
||||
'openai:gpt-5.5': { input: 4.00, output: 16.00 },
|
||||
|
||||
// ── Google ─────────────────────────────────────────────────────────────
|
||||
// `gemini-1.5-pro` was retired by Google (#3510); kept so historical
|
||||
// usage/audit rows still price. Not a valid default — it's deliberately
|
||||
// absent from the google recipe's chat list.
|
||||
'google:gemini-1.5-pro': { input: 1.25, output: 5.00 },
|
||||
// Gemini 2.0 Flash: $0.10 in / $0.40 out (verified 2026-06-03). Reconciled
|
||||
// from a stale $0.30/$1.20 entry that had drifted in takes-quality-eval.
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
/**
|
||||
* npm-squat-check — classify `gbrain` binaries found on PATH (#505).
|
||||
*
|
||||
* The npm registry name `gbrain` belongs to an unrelated third-party package;
|
||||
* this project is NOT distributed on npm. A reflexive `npm i -g gbrain` /
|
||||
* `bun add -g gbrain` therefore installs something that is not this project
|
||||
* and can shadow the real binary on PATH.
|
||||
*
|
||||
* Pure classification helpers (filesystem-only, no network, no shelling out)
|
||||
* so `gbrain doctor` can warn with receipts. The caller supplies the candidate
|
||||
* paths (typically the output of `which -a gbrain`).
|
||||
*/
|
||||
import { closeSync, openSync, readFileSync, readSync, realpathSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
export type GbrainBinaryKind = 'real' | 'foreign' | 'broken' | 'unknown';
|
||||
|
||||
export interface ClassifiedGbrainBinary {
|
||||
/** The candidate path as given (PATH entry / symlink). */
|
||||
path: string;
|
||||
kind: GbrainBinaryKind;
|
||||
/** Human-readable evidence for the classification. */
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface NpmSquatAssessment {
|
||||
status: 'ok' | 'warn' | 'skip';
|
||||
message: string;
|
||||
binaries: ClassifiedGbrainBinary[];
|
||||
}
|
||||
|
||||
/** Repository marker identifying this project's package.json. */
|
||||
const REAL_REPO_MARKER = 'garrytan/gbrain';
|
||||
|
||||
/** The documented install/remediation path, reused in doctor output. */
|
||||
export const NPM_SQUAT_REMEDIATION =
|
||||
`Remove the unrelated package (\`bun remove -g gbrain\` or \`npm uninstall -g gbrain\`) ` +
|
||||
`and install/upgrade only via the documented path: \`bun install -g github:${REAL_REPO_MARKER}\` ` +
|
||||
`(or \`git clone https://github.com/${REAL_REPO_MARKER}.git && bun install && bun link\`).`;
|
||||
|
||||
/**
|
||||
* A `bun build --compile` gbrain binary is a native executable, not a script.
|
||||
* Sniff the magic bytes: ELF, Mach-O (thin + fat), PE.
|
||||
*/
|
||||
function isNativeExecutable(path: string): boolean {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
fd = openSync(path, 'r');
|
||||
const buf = Buffer.alloc(4);
|
||||
if (readSync(fd, buf, 0, 4, 0) < 4) return false;
|
||||
const be = buf.readUInt32BE(0);
|
||||
const le = buf.readUInt32LE(0);
|
||||
return (
|
||||
be === 0x7f454c46 || // ELF
|
||||
be === 0xcafebabe || be === 0xcafebabf || // fat Mach-O
|
||||
le === 0xfeedface || le === 0xfeedfacf || // Mach-O 32/64
|
||||
(buf[0] === 0x4d && buf[1] === 0x5a) // PE ("MZ")
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
if (fd !== undefined) closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
/** Walk up from `start` to the nearest parseable package.json. */
|
||||
function nearestPackageJson(start: string): { dir: string; pkg: Record<string, any> } | null {
|
||||
let cur = start;
|
||||
for (let depth = 0; depth < 64; depth++) {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(join(cur, 'package.json'), 'utf8'));
|
||||
if (pkg && typeof pkg === 'object') return { dir: cur, pkg };
|
||||
} catch {
|
||||
// Missing or unparseable at this level; keep walking.
|
||||
}
|
||||
const parent = dirname(cur);
|
||||
if (parent === cur) break;
|
||||
cur = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this package.json THIS project? Two markers, either suffices:
|
||||
* - repository field pointing at garrytan/gbrain (string or { url }), or
|
||||
* - this repo's known bin shape (`"bin": { "gbrain": "src/cli.ts" }` — a
|
||||
* git checkout / `bun install -g github:...` install carries it verbatim;
|
||||
* a registry-published package ships built JS, not a bare .ts bin).
|
||||
*/
|
||||
function isRealGbrainPackage(pkg: Record<string, any>): boolean {
|
||||
const repo = typeof pkg.repository === 'string' ? pkg.repository : pkg.repository?.url;
|
||||
if (typeof repo === 'string' && repo.includes(REAL_REPO_MARKER)) return true;
|
||||
if (pkg.bin && typeof pkg.bin === 'object' && pkg.bin.gbrain === 'src/cli.ts') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify one candidate `gbrain` path:
|
||||
* - 'broken' : symlink that doesn't resolve / unreadable path.
|
||||
* - 'real' : compiled gbrain binary, or a script whose nearest
|
||||
* package.json is this project's (repo checkout / bun link /
|
||||
* `bun install -g github:garrytan/gbrain`).
|
||||
* - 'foreign' : nearest package.json is named "gbrain" but is NOT this
|
||||
* project — an unrelated registry install.
|
||||
* - 'unknown' : can't tell (no gbrain package.json above the resolved file).
|
||||
*/
|
||||
export function classifyGbrainBinary(path: string): ClassifiedGbrainBinary {
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = realpathSync(path);
|
||||
} catch {
|
||||
return { path, kind: 'broken', detail: 'broken symlink or unreadable path' };
|
||||
}
|
||||
if (isNativeExecutable(resolved)) {
|
||||
return { path, kind: 'real', detail: `compiled gbrain binary at ${resolved}` };
|
||||
}
|
||||
const found = nearestPackageJson(dirname(resolved));
|
||||
if (!found || found.pkg.name !== 'gbrain') {
|
||||
return { path, kind: 'unknown', detail: `no gbrain package.json found above ${resolved}` };
|
||||
}
|
||||
if (isRealGbrainPackage(found.pkg)) {
|
||||
return { path, kind: 'real', detail: `this project's install at ${found.dir}` };
|
||||
}
|
||||
return {
|
||||
path,
|
||||
kind: 'foreign',
|
||||
detail: `unrelated npm package named "gbrain" at ${found.dir}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Assess candidate paths in PATH precedence order (first entry wins when the
|
||||
* shell runs `gbrain`).
|
||||
*
|
||||
* - skip : no candidates (gbrain not on PATH — nothing to check).
|
||||
* - warn : the winning entry is broken, or an unrelated npm package shadows
|
||||
* (appears before) the real binary — including when no real binary
|
||||
* is on PATH at all.
|
||||
* - ok : the winning entry is the real binary (an unrelated install
|
||||
* sitting BEHIND it is noted but not a warn).
|
||||
*/
|
||||
export function assessGbrainBinaries(candidates: string[]): NpmSquatAssessment {
|
||||
const unique = [...new Set(candidates.map((c) => c.trim()).filter(Boolean))];
|
||||
if (unique.length === 0) {
|
||||
return { status: 'skip', message: 'gbrain not found on PATH', binaries: [] };
|
||||
}
|
||||
const binaries = unique.map(classifyGbrainBinary);
|
||||
const first = binaries[0]!;
|
||||
const realIdx = binaries.findIndex((b) => b.kind === 'real');
|
||||
const foreignIdx = binaries.findIndex((b) => b.kind === 'foreign');
|
||||
|
||||
if (first.kind === 'broken') {
|
||||
return {
|
||||
status: 'warn',
|
||||
message:
|
||||
`\`gbrain\` on PATH is a broken link (${first.path}). ` +
|
||||
`Note: gbrain is NOT distributed on npm — the npm package named "gbrain" is unrelated. ` +
|
||||
NPM_SQUAT_REMEDIATION,
|
||||
binaries,
|
||||
};
|
||||
}
|
||||
if (foreignIdx !== -1 && (realIdx === -1 || foreignIdx < realIdx)) {
|
||||
const foreign = binaries[foreignIdx]!;
|
||||
return {
|
||||
status: 'warn',
|
||||
message:
|
||||
`\`gbrain\` on PATH resolves to an unrelated npm package, not this project ` +
|
||||
`(${foreign.path} — ${foreign.detail}). gbrain is NOT distributed on npm. ` +
|
||||
NPM_SQUAT_REMEDIATION,
|
||||
binaries,
|
||||
};
|
||||
}
|
||||
if (foreignIdx !== -1) {
|
||||
return {
|
||||
status: 'ok',
|
||||
message:
|
||||
`real gbrain wins on PATH (${first.path}), but an unrelated npm package named ` +
|
||||
`"gbrain" is also installed (${binaries[foreignIdx]!.path}). Consider removing it: ` +
|
||||
`\`bun remove -g gbrain\` / \`npm uninstall -g gbrain\`.`,
|
||||
binaries,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'ok',
|
||||
message:
|
||||
first.kind === 'real'
|
||||
? `gbrain on PATH is the real binary (${first.path}).`
|
||||
: `no unrelated npm "gbrain" install detected on PATH (${first.path}).`,
|
||||
binaries,
|
||||
};
|
||||
}
|
||||
+99
-253
@@ -11,7 +11,7 @@ import type { GBrainConfig } from './config.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import { importFromContent } from './import-file.ts';
|
||||
import { writePageThrough } from './write-through.ts';
|
||||
import { hybridSearch, hybridSearchCached, stampContentFlags, stampUnverifiedExtractions } from './search/hybrid.ts';
|
||||
import { hybridSearch, hybridSearchCached, stampContentFlags } from './search/hybrid.ts';
|
||||
import { expandQuery } from './search/expansion.ts';
|
||||
import { dedupResults } from './search/dedup.ts';
|
||||
import { captureEvalCandidate, isEvalCaptureEnabled, isEvalScrubEnabled } from './eval-capture.ts';
|
||||
@@ -21,14 +21,11 @@ import { isFactsBackstopEligible } from './facts/eligibility.ts';
|
||||
import { stripTakesFence } from './takes-fence.ts';
|
||||
import { stripFactsFence } from './facts-fence.ts';
|
||||
import { getContentFlag } from './quarantine.ts';
|
||||
import { unverifiedExtractionFragment, isUnverifiedExtraction, EXTRACTION_STATUS_KEY, STATUS_VERIFIED } from './extraction-review.ts';
|
||||
import { buildVisibilityClause } from './search/sql-ranking.ts';
|
||||
import { bumpLastRetrievedAt } from './last-retrieved.ts';
|
||||
import { isSearchMode } from './search/mode.ts';
|
||||
import { stampEvidence } from './search/evidence.ts';
|
||||
import type { SearchResult } from './types.ts';
|
||||
import { CJK_SLUG_CHARS, PAGE_SLUG_SEG } from './cjk.ts';
|
||||
import { ALL_SOURCES } from './source-id.ts';
|
||||
import * as db from './db.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import {
|
||||
@@ -163,11 +160,10 @@ export function validatePageSlug(slug: string): void {
|
||||
if (slug.length > 255) {
|
||||
throw new OperationError('invalid_params', 'page_slug exceeds 255 characters');
|
||||
}
|
||||
// #3417: letters/numbers from any script allowed in segments (u flag required
|
||||
// for the \p{...} classes in PAGE_SLUG_SEG). Shape rules (lead char, hyphen
|
||||
// continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'iu').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: letters/numbers in any script, hyphens, forward-slash separated segments)`);
|
||||
// v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed
|
||||
// in segments. ASCII shape rules (lead char, hyphen continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -488,14 +484,6 @@ export function sourceScopeOpts(ctx: OperationContext): { sourceId?: string; sou
|
||||
// value of `[]` MUST NOT widen scope to "all sources" by being interpreted
|
||||
// as "no filter."
|
||||
if (allowed && allowed.length > 0) return { sourceIds: allowed };
|
||||
// #1712: the __all__ sentinel spans the brain — but ONLY for trusted local
|
||||
// callers (strictly `remote === false`). For remote/untrusted callers the
|
||||
// literal stays as-is: it can never match a real source id (underscores are
|
||||
// rejected at creation), so the read fail-closes to empty rather than
|
||||
// widening past the caller's grant. Do NOT "simplify" this to `{}`.
|
||||
if (ctx.sourceId === ALL_SOURCES) {
|
||||
return ctx.remote === false ? {} : { sourceId: ctx.sourceId };
|
||||
}
|
||||
if (ctx.sourceId) return { sourceId: ctx.sourceId };
|
||||
return {};
|
||||
}
|
||||
@@ -563,7 +551,7 @@ export function resolveRequestedScope(
|
||||
sourceIdParam: string | undefined,
|
||||
allSourcesParam = false,
|
||||
): { sourceId?: string; sourceIds?: string[] } {
|
||||
const wantsAll = allSourcesParam || sourceIdParam === ALL_SOURCES;
|
||||
const wantsAll = allSourcesParam || sourceIdParam === '__all__';
|
||||
if (wantsAll) {
|
||||
return ctx.remote === false ? {} : sourceScopeOpts(ctx);
|
||||
}
|
||||
@@ -1196,9 +1184,7 @@ const put_page: Operation = {
|
||||
let writerLint: { error_count: number; warning_count: number } | { skipped: string } | undefined;
|
||||
try {
|
||||
const { runPostWriteLint } = await import('./output/post-write.ts');
|
||||
const lint = await runPostWriteLint(ctx.engine, result.slug, {
|
||||
sourceId: ctx.sourceId ?? 'default',
|
||||
});
|
||||
const lint = await runPostWriteLint(ctx.engine, result.slug);
|
||||
if (lint.ran) {
|
||||
writerLint = {
|
||||
error_count: lint.findings.filter(f => f.severity === 'error').length,
|
||||
@@ -1639,10 +1625,6 @@ const search: Operation = {
|
||||
// agent-warning channel (hybridSearch stamps it; this branch bypasses
|
||||
// hybridSearch, so stamp explicitly). Fail-open inside the helper.
|
||||
await stampContentFlags(ctx.engine, results);
|
||||
// #160: same for the unverified auto-extracted stub marker (no boost
|
||||
// to cancel on this path — keyword-only never applies the compiled-
|
||||
// truth boost — but the provenance marker must still surface).
|
||||
await stampUnverifiedExtractions(ctx.engine, results);
|
||||
bumpLastRetrievedAt(ctx.engine, results.map((r) => r.page_id));
|
||||
maybeCaptureSearch(ctx, queryText, results, Date.now() - startedAt, false);
|
||||
return results;
|
||||
@@ -5043,7 +5025,7 @@ const schema_review_orphans: Operation = {
|
||||
|
||||
const schema_apply_mutations: Operation = {
|
||||
name: 'schema_apply_mutations',
|
||||
description: 'v0.40.7.0: batched schema pack mutation. ATOMIC: every mutation is validated against an in-memory manifest first, and the pack file is written to disk at most once, after the FULL batch has proven valid — so a failure at any point leaves the pack file byte-identical to its pre-batch state (never a partial write). Audit log records one batch_id. Admin scope; NOT localOnly so remote agents (your OpenClaw, etc.) can author packs over normal MCP. Mutation shape per ApplyMutationsRequest type — supports add_type / remove_type / update_type / add_alias / remove_alias / add_prefix / remove_prefix / add_link_type / remove_link_type / set_extractable / set_expert_routing.',
|
||||
description: 'v0.40.7.0: batched schema pack mutation. ATOMIC: all mutations succeed or all roll back. Audit log records one batch_id. Admin scope; NOT localOnly so remote agents (your OpenClaw, etc.) can author packs over normal MCP. Mutation shape per ApplyMutationsRequest type — supports add_type / remove_type / update_type / add_alias / remove_alias / add_prefix / remove_prefix / add_link_type / remove_link_type / set_extractable / set_expert_routing.',
|
||||
params: {
|
||||
pack: { type: 'string', required: true, description: 'Pack to mutate (must not be bundled)' },
|
||||
mutations: {
|
||||
@@ -5066,20 +5048,92 @@ const schema_apply_mutations: Operation = {
|
||||
const batchId = `batch-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const actor = ctx.auth?.clientId ? `mcp:${ctx.auth.clientId.slice(0, 8)}` : 'cli';
|
||||
const sourceId = ctx.sourceId; // codex C5: write-side scoping
|
||||
// `applyMutationsAtomic` (issue #2581) owns the lock + single read +
|
||||
// single write for the whole batch: every mutation is validated
|
||||
// in-memory first, and the pack file is written at most once, only
|
||||
// after the FULL batch checks out. That is what makes this actually
|
||||
// atomic (a failure at any index can never leave earlier mutations on
|
||||
// disk), vs. the old per-mutation-writes-as-it-goes shape.
|
||||
const { applyMutationsAtomic } = await import('./schema-pack/mutate.ts');
|
||||
// Compose every mutation inside ONE withPackLock so the batch is
|
||||
// truly atomic. The withMutation skeleton handles audit / cache
|
||||
// invalidation per operation; we orchestrate the lock + iteration.
|
||||
const { withPackLock } = await import('./schema-pack/pack-lock.ts');
|
||||
const {
|
||||
addTypeToPack, removeTypeFromPack, updateTypeOnPack,
|
||||
addAliasToType, removeAliasFromType, addPrefixToType, removePrefixFromType,
|
||||
addLinkTypeToPack, removeLinkTypeFromPack,
|
||||
setExtractableOnType, setExpertRoutingOnType,
|
||||
SchemaPackMutationError,
|
||||
} = await import('./schema-pack/mutate.ts');
|
||||
const baseMutateOpts = {
|
||||
actor: actor as 'cli' | `mcp:${string}`,
|
||||
batchId,
|
||||
engine: ctx.engine,
|
||||
...(sourceId ? { sourceId } : {}),
|
||||
...(force ? { force: true } : {}),
|
||||
};
|
||||
const results: unknown[] = [];
|
||||
try {
|
||||
const results = await applyMutationsAtomic(pack, mutations, {
|
||||
actor: actor as 'cli' | `mcp:${string}`,
|
||||
batchId,
|
||||
engine: ctx.engine,
|
||||
...(sourceId ? { sourceId } : {}),
|
||||
...(force ? { force: true } : {}),
|
||||
// Outer lock: hold the pack for the whole batch so other writers
|
||||
// can't slip in between mutations.
|
||||
await withPackLock(pack, { force, lockDir: undefined }, async () => {
|
||||
for (let i = 0; i < mutations.length; i++) {
|
||||
const m = mutations[i]!;
|
||||
// Each primitive acquires the lock internally; the outer
|
||||
// withPackLock makes that re-entrant via fast-stale-detect
|
||||
// (--force option for the inner call). To keep semantics
|
||||
// simple, we pass {force:true} to the inner calls because
|
||||
// they're nested inside our outer lock — we already own it.
|
||||
const innerOpts = { ...baseMutateOpts, force: true };
|
||||
let r: unknown;
|
||||
switch (m.op) {
|
||||
case 'add_type':
|
||||
r = await addTypeToPack(pack, {
|
||||
name: m.name as string,
|
||||
primitive: m.primitive as never,
|
||||
prefix: m.prefix as string,
|
||||
extractable: m.extractable as boolean | undefined,
|
||||
expertRouting: m.expert_routing as boolean | undefined,
|
||||
aliases: m.aliases as string[] | undefined,
|
||||
}, innerOpts);
|
||||
break;
|
||||
case 'remove_type':
|
||||
r = await removeTypeFromPack(pack, m.name as string, innerOpts);
|
||||
break;
|
||||
case 'update_type':
|
||||
r = await updateTypeOnPack(pack, { name: m.name as string, patch: (m.patch as object) ?? {} }, innerOpts);
|
||||
break;
|
||||
case 'add_alias':
|
||||
r = await addAliasToType(pack, m.type as string, m.alias as string, innerOpts);
|
||||
break;
|
||||
case 'remove_alias':
|
||||
r = await removeAliasFromType(pack, m.type as string, m.alias as string, innerOpts);
|
||||
break;
|
||||
case 'add_prefix':
|
||||
r = await addPrefixToType(pack, m.type as string, m.prefix as string, innerOpts);
|
||||
break;
|
||||
case 'remove_prefix':
|
||||
r = await removePrefixFromType(pack, m.type as string, m.prefix as string, innerOpts);
|
||||
break;
|
||||
case 'add_link_type':
|
||||
r = await addLinkTypeToPack(pack, {
|
||||
name: m.name as string,
|
||||
inverse: m.inverse as string | undefined,
|
||||
inference: m.inference as { regex?: string; page_type?: string; target_type?: string } | undefined,
|
||||
}, innerOpts);
|
||||
break;
|
||||
case 'remove_link_type':
|
||||
r = await removeLinkTypeFromPack(pack, m.name as string, innerOpts);
|
||||
break;
|
||||
case 'set_extractable':
|
||||
r = await setExtractableOnType(pack, m.type as string, m.value as boolean, innerOpts);
|
||||
break;
|
||||
case 'set_expert_routing':
|
||||
r = await setExpertRoutingOnType(pack, m.type as string, m.value as boolean, innerOpts);
|
||||
break;
|
||||
default:
|
||||
throw new SchemaPackMutationError(
|
||||
'INVALID_RESULT',
|
||||
`unknown mutation op: '${m.op}' at index ${i}`,
|
||||
{ index: i, op: m.op },
|
||||
);
|
||||
}
|
||||
results.push({ index: i, op: m.op, ...(r as object) });
|
||||
}
|
||||
});
|
||||
return {
|
||||
schema_version: 1,
|
||||
@@ -5090,21 +5144,17 @@ const schema_apply_mutations: Operation = {
|
||||
};
|
||||
} catch (e) {
|
||||
const code = (e as { code?: string }).code ?? 'UNKNOWN';
|
||||
const failedAtIndex = (e as { details?: { index?: number } }).details?.index;
|
||||
return {
|
||||
error: 'mutation_failed',
|
||||
code,
|
||||
message: (e as Error).message,
|
||||
batch_id: batchId,
|
||||
// Nothing was written to disk — applyMutationsAtomic only writes
|
||||
// once, after every mutation in the batch has validated cleanly.
|
||||
// (Pre-fix, this field was `partial_results` and listed mutations
|
||||
// that HAD already landed on disk, because the old implementation
|
||||
// wrote as it went — that shape is gone; a failed batch can no
|
||||
// longer imply partial application.)
|
||||
mutations_applied: 0,
|
||||
pack_unchanged: true,
|
||||
...(failedAtIndex !== undefined ? { failed_at_index: failedAtIndex } : {}),
|
||||
// Partial results recorded so the agent can inspect which
|
||||
// mutations landed before the failure (the atomic guarantee
|
||||
// is at the LOCK level — individual mutations are sequential
|
||||
// and each is atomic; pack state reflects everything up to the
|
||||
// failed mutation).
|
||||
partial_results: results,
|
||||
};
|
||||
}
|
||||
},
|
||||
@@ -5621,208 +5671,6 @@ const chronicle_backfill: Operation = {
|
||||
cliHints: { name: 'chronicle-backfill' },
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extraction quarantine lane (issue #160)
|
||||
//
|
||||
// `extractAndEnrich` regex-extracts entity names from arbitrary text and
|
||||
// creates people/ + companies/ stub pages. These three ops are its ONLY
|
||||
// sanctioned surface:
|
||||
// - extract_entities — run extraction. Direct authoritative writes need
|
||||
// BOTH the trusted local CLI (ctx.remote === false)
|
||||
// AND the explicit --trusted-extraction flag;
|
||||
// everything else lands in the quarantine lane
|
||||
// (frontmatter provenance/status markers).
|
||||
// - extraction_pending — list unverified stubs awaiting review.
|
||||
// - extraction_review — promote (status → verified) or reject
|
||||
// (soft-delete) in batch. Owner-only (fail-closed
|
||||
// on ctx.remote): THIS surface never lets a remote
|
||||
// caller flip the status markers. Scope note: the
|
||||
// markers are ordinary frontmatter, so a caller who
|
||||
// already holds generic remote put_page write scope
|
||||
// can rewrite the page (markers included) — that
|
||||
// caller could equally author an unmarked people/
|
||||
// page directly, so the lane adds no privilege
|
||||
// there; put_page authz is its own boundary.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Resource guards for extract_entities (#160 hardening): bound the work a
|
||||
// single remote write-scope call can trigger. ponytail: flat caps; make them
|
||||
// config knobs only if a real workload hits them.
|
||||
const MAX_EXTRACT_TEXT_CHARS = 200_000;
|
||||
const MAX_EXTRACT_ENTITIES = 200;
|
||||
|
||||
const extract_entities: Operation = {
|
||||
name: 'extract_entities',
|
||||
description: 'Extract entity names (people, companies) from text and create/update their brain stub pages. Stubs from untrusted input land in the quarantine lane (frontmatter `provenance: auto-extracted` + `status: unverified`) — excluded from authoritative retrieval boosts until reviewed. Direct authoritative writes require the trusted local CLI AND --trusted-extraction.',
|
||||
params: {
|
||||
text: { type: 'string', required: true, description: 'The text to extract entities from (email, transcript, pasted content, …). Max 200k characters — split larger inputs.' },
|
||||
source_slug: { type: 'string', required: true, description: 'Slug of the source page the text came from (used for backlinks + timeline attribution).' },
|
||||
trusted_extraction: { type: 'boolean', required: false, description: 'Local CLI only: write stubs directly as authoritative pages, skipping the quarantine lane. Ignored (always quarantined) for remote callers.' },
|
||||
},
|
||||
mutating: true,
|
||||
scope: 'write',
|
||||
handler: async (ctx, p) => {
|
||||
// Trust rule (#160, fail-closed like the CV6 provenance gate above):
|
||||
// `ctx.remote === false` is the ONLY truthy condition that can admit a
|
||||
// direct authoritative write, and even then the caller must opt in
|
||||
// explicitly. Remote/unset trust → quarantine lane, flag ignored.
|
||||
const trusted = ctx.remote === false && p.trusted_extraction === true;
|
||||
const text = p.text as string;
|
||||
// Resource guards: the greedy name regex on a huge paste can yield tens
|
||||
// of thousands of "entities", each costing several DB round-trips. Cap
|
||||
// input size loudly and entity count softly (surfaced as `truncated`).
|
||||
if (text.length > MAX_EXTRACT_TEXT_CHARS) {
|
||||
throw new OperationError(
|
||||
'invalid_params',
|
||||
`extract_entities: text is ${text.length} chars (max ${MAX_EXTRACT_TEXT_CHARS}).`,
|
||||
'Split the input and call extract_entities per section.',
|
||||
);
|
||||
}
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'extract_entities', trusted };
|
||||
const { extractEntities, enrichEntities } = await import('./enrichment-service.ts');
|
||||
const found = extractEntities(text);
|
||||
const capped = found.slice(0, MAX_EXTRACT_ENTITIES);
|
||||
const results = await enrichEntities(
|
||||
ctx.engine,
|
||||
capped.map((e) => ({ entityName: e.name, entityType: e.type, context: e.context, sourceSlug: p.source_slug as string })),
|
||||
{
|
||||
trusted,
|
||||
...(ctx.sourceId ? { sourceId: ctx.sourceId } : {}),
|
||||
// Pure local DB writes — no external API call to pace, so the
|
||||
// system-load capacity gate would only stall the caller.
|
||||
throttle: false,
|
||||
},
|
||||
);
|
||||
return {
|
||||
status: 'ok',
|
||||
trusted,
|
||||
quarantined: results.filter((r) => r.quarantined === true).length,
|
||||
count: results.length,
|
||||
entities_found: found.length,
|
||||
truncated: found.length > capped.length,
|
||||
entities: results,
|
||||
};
|
||||
},
|
||||
cliHints: { name: 'extract-entities' },
|
||||
};
|
||||
|
||||
const extraction_pending: Operation = {
|
||||
name: 'extraction_pending',
|
||||
description: 'List unverified auto-extracted entity stubs awaiting owner review (the quarantine lane from extract_entities). Promote or reject them with extraction_review.',
|
||||
params: {
|
||||
limit: { type: 'number', required: false, description: 'Max rows (default 100, cap 500).' },
|
||||
offset: { type: 'number', required: false, description: 'Pagination offset.' },
|
||||
},
|
||||
scope: 'read',
|
||||
handler: async (ctx, p) => {
|
||||
const limit = Math.min(Math.max(Number(p.limit ?? 100) || 100, 1), 500);
|
||||
const offset = Math.max(Number(p.offset ?? 0) || 0, 0);
|
||||
// Read-side source isolation: route through sourceScopeOpts (federated
|
||||
// array > scalar > nothing), applied in SQL below.
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
const params: unknown[] = [];
|
||||
let srcClause = '';
|
||||
if (scope.sourceIds && scope.sourceIds.length > 0) {
|
||||
params.push(scope.sourceIds);
|
||||
srcClause = `AND p.source_id = ANY($${params.length}::text[])`;
|
||||
} else if (scope.sourceId) {
|
||||
params.push(scope.sourceId);
|
||||
srcClause = `AND p.source_id = $${params.length}`;
|
||||
}
|
||||
params.push(limit, offset);
|
||||
const rows = await ctx.engine.executeRaw<{
|
||||
slug: string; title: string; type: string; source_id: string;
|
||||
extracted_from: string | null; created_at: string;
|
||||
}>(
|
||||
`SELECT p.slug, p.title, p.type, p.source_id,
|
||||
p.frontmatter ->> 'source' AS extracted_from,
|
||||
p.created_at::text AS created_at
|
||||
FROM pages p
|
||||
JOIN sources s ON s.id = p.source_id
|
||||
WHERE ${unverifiedExtractionFragment('p')}
|
||||
${buildVisibilityClause('p', 's')}
|
||||
${srcClause}
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT $${params.length - 1} OFFSET $${params.length}`,
|
||||
params,
|
||||
);
|
||||
return { count: rows.length, pending: rows };
|
||||
},
|
||||
cliHints: { name: 'extraction-pending' },
|
||||
};
|
||||
|
||||
const extraction_review: Operation = {
|
||||
name: 'extraction_review',
|
||||
description: 'Promote or reject unverified auto-extracted entity stubs (batch). Promote flips `status` to verified (provenance kept for audit); reject soft-deletes the stub. Owner-only: this op is refused for any non-local caller. (The markers are ordinary frontmatter — the boundary against rewriting them wholesale is put_page write authz, same as for any page.)',
|
||||
params: {
|
||||
action: { type: 'string', required: true, description: "'promote' or 'reject'." },
|
||||
slugs: { type: 'array', required: true, items: { type: 'string' }, description: 'Stub slugs to act on (batch).' },
|
||||
},
|
||||
mutating: true,
|
||||
scope: 'write',
|
||||
localOnly: true,
|
||||
handler: async (ctx, p) => {
|
||||
// The review decision IS the trust gate — if a remote caller could
|
||||
// promote, injected content could self-promote and the quarantine lane
|
||||
// would be decorative. Fail-closed: only strictly-local callers pass.
|
||||
if (ctx.remote !== false) {
|
||||
throw new OperationError(
|
||||
'permission_denied',
|
||||
'extraction_review is owner-only: promote/reject decisions must come from the trusted local CLI.',
|
||||
'Run `gbrain extraction-review <promote|reject> --slugs ...` on the host machine.',
|
||||
);
|
||||
}
|
||||
const action = p.action as string;
|
||||
if (action !== 'promote' && action !== 'reject') {
|
||||
throw new OperationError('invalid_params', `extraction_review: action must be 'promote' or 'reject'; got '${action}'.`);
|
||||
}
|
||||
// CLI passes `--slugs a,b,c` as one string; MCP passes a real array.
|
||||
const slugs = Array.isArray(p.slugs)
|
||||
? (p.slugs as string[])
|
||||
: typeof p.slugs === 'string'
|
||||
? p.slugs.split(',').map((s) => s.trim()).filter(Boolean)
|
||||
: [];
|
||||
if (slugs.length === 0) {
|
||||
throw new OperationError('invalid_params', 'extraction_review: slugs must be a non-empty array (CLI: --slugs slug1,slug2).');
|
||||
}
|
||||
if (ctx.dryRun) return { dry_run: true, action: `extraction_review:${action}`, slugs };
|
||||
const results: Array<{ slug: string; status: string }> = [];
|
||||
for (const slug of slugs) {
|
||||
const page = await ctx.engine.getPage(slug, ctx.sourceId ? { sourceId: ctx.sourceId } : undefined);
|
||||
if (!page) {
|
||||
results.push({ slug, status: 'not_found' });
|
||||
continue;
|
||||
}
|
||||
if (!isUnverifiedExtraction(page.frontmatter)) {
|
||||
results.push({ slug, status: 'not_unverified' });
|
||||
continue;
|
||||
}
|
||||
if (action === 'promote') {
|
||||
// Frontmatter-only flip via a targeted JSONB merge — NOT putPage,
|
||||
// whose upsert would reset non-carried columns (page_kind →
|
||||
// 'markdown', content_hash, …) for a change that only touches one
|
||||
// frontmatter key. provenance stays 'auto-extracted' as the audit
|
||||
// trail of HOW the page came to exist; status → 'verified' records
|
||||
// the owner's call. jsonb_build_object binds as text (no
|
||||
// JSON.stringify-into-::jsonb hazard); identical on both engines.
|
||||
await ctx.engine.executeRaw(
|
||||
`UPDATE pages
|
||||
SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || jsonb_build_object($1::text, $2::text),
|
||||
updated_at = now()
|
||||
WHERE slug = $3 AND source_id = $4`,
|
||||
[EXTRACTION_STATUS_KEY, STATUS_VERIFIED, slug, page.source_id],
|
||||
);
|
||||
results.push({ slug, status: 'promoted' });
|
||||
} else {
|
||||
await ctx.engine.softDeletePage(slug, { sourceId: page.source_id });
|
||||
results.push({ slug, status: 'rejected' });
|
||||
}
|
||||
}
|
||||
return { status: 'ok', action, results };
|
||||
},
|
||||
cliHints: { name: 'extraction-review', positional: ['action'] },
|
||||
};
|
||||
|
||||
export const operations: Operation[] = [
|
||||
// Page CRUD
|
||||
get_page, put_page, delete_page, list_pages,
|
||||
@@ -5879,8 +5727,6 @@ export const operations: Operation[] = [
|
||||
volunteer_chronicle, chronicle_backfill,
|
||||
// v0.43 (#2095): push-based context
|
||||
volunteer_context,
|
||||
// Extraction quarantine lane (#160): gated entity extraction + review queue
|
||||
extract_entities, extraction_pending, extraction_review,
|
||||
// v0.31: hot memory (facts table)
|
||||
extract_facts, recall, forget_fact,
|
||||
// v0.32.6: contradiction probe MCP surface (M3)
|
||||
|
||||
@@ -38,10 +38,6 @@ export interface PostWriteLintOpts {
|
||||
force?: boolean;
|
||||
/** Skip file writes; used by tests. */
|
||||
noLog?: boolean;
|
||||
/** Exact scalar source for the page and nested validation reads. */
|
||||
sourceId?: string;
|
||||
/** Federated read scope; when non-empty, takes precedence over sourceId. */
|
||||
sourceIds?: string[];
|
||||
}
|
||||
|
||||
export interface PostWriteLintResult {
|
||||
@@ -84,12 +80,7 @@ export async function runPostWriteLint(
|
||||
return { ran: false, slug, findings: [], skippedReason: 'flag_disabled' };
|
||||
}
|
||||
|
||||
const sourceOpts = opts.sourceIds && opts.sourceIds.length > 0
|
||||
? { sourceIds: opts.sourceIds }
|
||||
: opts.sourceId
|
||||
? { sourceId: opts.sourceId }
|
||||
: undefined;
|
||||
const page = await engine.getPage(slug, sourceOpts);
|
||||
const page = await engine.getPage(slug);
|
||||
if (!page) {
|
||||
return { ran: false, slug, findings: [], skippedReason: 'page_not_found' };
|
||||
}
|
||||
@@ -106,8 +97,6 @@ export async function runPostWriteLint(
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter ?? {},
|
||||
engine,
|
||||
sourceId: opts.sourceId,
|
||||
sourceIds: opts.sourceIds,
|
||||
};
|
||||
|
||||
const findings: ValidationFinding[] = [];
|
||||
|
||||
@@ -72,10 +72,9 @@ export class SlugRegistryError extends Error {
|
||||
// SlugRegistry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Shares the page-slug segment grammar (all scripts, #738/#3417) with
|
||||
// Shares the page-slug segment grammar (incl. CJK ranges, #738) with
|
||||
// validatePageSlug; keeps this site's dir/name shape (>= 2 segments).
|
||||
// `u` flag required by PAGE_SLUG_SEG's \p{...} classes.
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u');
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`);
|
||||
|
||||
export class SlugRegistry {
|
||||
constructor(private engine: BrainEngine) {}
|
||||
|
||||
@@ -23,46 +23,25 @@ export const backLinkValidator: PageValidator = {
|
||||
|
||||
async validate(ctx: PageValidationContext): Promise<ValidationFinding[]> {
|
||||
const findings: ValidationFinding[] = [];
|
||||
const federatedSourceIds = ctx.sourceIds && ctx.sourceIds.length > 0
|
||||
? ctx.sourceIds
|
||||
: undefined;
|
||||
const outboundOpts = federatedSourceIds
|
||||
? { sourceIds: federatedSourceIds }
|
||||
: ctx.sourceId
|
||||
? { sourceId: ctx.sourceId }
|
||||
: undefined;
|
||||
|
||||
const outbound = await ctx.engine.getLinks(ctx.slug, outboundOpts);
|
||||
const outbound = await ctx.engine.getLinks(ctx.slug);
|
||||
if (outbound.length === 0) return findings;
|
||||
|
||||
// A federated lookup can return same-slug origins and targets from several
|
||||
// sources. Deduplicate only identical endpoint pairs; every distinct origin
|
||||
// still needs its own exact reverse.
|
||||
const uniqueEdges = new Map<string, typeof outbound[number]>();
|
||||
for (const link of outbound) {
|
||||
uniqueEdges.set(
|
||||
`${link.from_source_id}\0${link.from_slug}\0${link.to_source_id}\0${link.to_slug}`,
|
||||
link,
|
||||
);
|
||||
}
|
||||
// Iron Law: if ctx.slug → target, target must ALSO link back to ctx.slug.
|
||||
// We check target's outbound links; if none of them point at ctx.slug,
|
||||
// the back-link is missing.
|
||||
const uniqueTargets = new Set<string>();
|
||||
for (const link of outbound) uniqueTargets.add(link.to_slug);
|
||||
|
||||
for (const target of uniqueEdges.values()) {
|
||||
const targetOpts = federatedSourceIds
|
||||
? { sourceIds: federatedSourceIds }
|
||||
: { sourceId: target.to_source_id };
|
||||
const targetOutbound = await ctx.engine.getLinks(target.to_slug, targetOpts);
|
||||
const hasReverse = targetOutbound.some(link =>
|
||||
link.from_source_id === target.to_source_id
|
||||
&& link.from_slug === target.to_slug
|
||||
&& link.to_source_id === target.from_source_id
|
||||
&& link.to_slug === target.from_slug
|
||||
);
|
||||
for (const target of uniqueTargets) {
|
||||
const targetOutbound = await ctx.engine.getLinks(target);
|
||||
const hasReverse = targetOutbound.some(l => l.to_slug === ctx.slug);
|
||||
if (!hasReverse) {
|
||||
findings.push({
|
||||
slug: ctx.slug,
|
||||
validator: 'back-link',
|
||||
severity: 'warning',
|
||||
message: `Outbound link to ${target.to_slug} has no back-link (${target.to_slug} does not reference ${ctx.slug}). runAutoLink should reconcile this on next put_page; flag for inspection.`,
|
||||
message: `Outbound link to ${target} has no back-link (${target} does not reference ${ctx.slug}). runAutoLink should reconcile this on next put_page; flag for inspection.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,14 +62,9 @@ export const linkValidator: PageValidator = {
|
||||
linkPositions.set(slug, list);
|
||||
}
|
||||
|
||||
// Batch-check which targets exist within the validation read scope.
|
||||
const sourceOpts = ctx.sourceIds && ctx.sourceIds.length > 0
|
||||
? { sourceIds: ctx.sourceIds }
|
||||
: ctx.sourceId
|
||||
? { sourceId: ctx.sourceId }
|
||||
: undefined;
|
||||
// Batch-check which targets exist.
|
||||
for (const slug of internalTargets) {
|
||||
const page = await ctx.engine.getPage(slug, sourceOpts);
|
||||
const page = await ctx.engine.getPage(slug);
|
||||
if (page) continue;
|
||||
const positions = linkPositions.get(slug) ?? [];
|
||||
for (const pos of positions) {
|
||||
|
||||
@@ -93,10 +93,6 @@ export interface PageValidationContext {
|
||||
timeline: string;
|
||||
frontmatter: Record<string, unknown>;
|
||||
engine: BrainEngine;
|
||||
/** Exact scalar source for source-qualified validation reads. */
|
||||
sourceId?: string;
|
||||
/** Federated read scope; when non-empty, takes precedence over sourceId. */
|
||||
sourceIds?: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -253,9 +249,7 @@ export class BrainWriter {
|
||||
|
||||
// Validators run before the outer transaction commits.
|
||||
if (strict !== 'off') {
|
||||
report = await runValidators(txEngine, validators, tx.touchedSlugs, {
|
||||
sourceId: 'default',
|
||||
});
|
||||
report = await runValidators(txEngine, validators, tx.touchedSlugs);
|
||||
// `ctx.logger.info` would be nice but keep validator behavior uniform
|
||||
// regardless of strict/lint mode. Caller inspects the report.
|
||||
if (strict === 'strict' && report.errorCount > 0) {
|
||||
@@ -287,17 +281,11 @@ async function runValidators(
|
||||
engine: BrainEngine,
|
||||
validators: PageValidator[],
|
||||
touchedSlugs: Set<string>,
|
||||
scope: { sourceId?: string; sourceIds?: string[] } = {},
|
||||
): Promise<ValidationReport> {
|
||||
const findings: ValidationFinding[] = [];
|
||||
const sourceOpts = scope.sourceIds && scope.sourceIds.length > 0
|
||||
? { sourceIds: scope.sourceIds }
|
||||
: scope.sourceId
|
||||
? { sourceId: scope.sourceId }
|
||||
: undefined;
|
||||
|
||||
for (const slug of touchedSlugs) {
|
||||
const page = await engine.getPage(slug, sourceOpts);
|
||||
const page = await engine.getPage(slug);
|
||||
if (!page) continue; // could have been deleted in this tx
|
||||
|
||||
// Grandfather opt-out
|
||||
@@ -310,8 +298,6 @@ async function runValidators(
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter ?? {},
|
||||
engine,
|
||||
sourceId: scope.sourceId,
|
||||
sourceIds: scope.sourceIds,
|
||||
};
|
||||
|
||||
for (const v of validators) {
|
||||
|
||||
+38
-135
@@ -17,26 +17,7 @@ import type {
|
||||
SourceRow,
|
||||
} from './engine.ts';
|
||||
import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts';
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
|
||||
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { PGLITE_SCHEMA_SQL, getPGLiteSchema } from './pglite-schema.ts';
|
||||
@@ -77,7 +58,6 @@ import { finalizeLastSeen } from './chronicle/last-seen.ts';
|
||||
import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from './extraction-review.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
import {
|
||||
@@ -438,13 +418,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
let dims: number = DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
|
||||
await this.db.exec(getPGLiteSchema(dims, model));
|
||||
@@ -1002,8 +978,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
|
||||
params
|
||||
);
|
||||
@@ -2097,10 +2072,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Built on the bare `slug` output column: applied inside the `scored` CTE
|
||||
// whose FROM is the single relation `hnsw_candidates`, so unqualified
|
||||
// `slug` resolves cleanly (T1 per-page pool restructure).
|
||||
// issue #160: guard predicate projected as `unverified_stub` in
|
||||
// hnsw_candidates (parity with postgres-engine) so unverified stubs get
|
||||
// factor 1.0, not the people/ 1.2x, inside the pre-LIMIT re-rank.
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail, 'unverified_stub');
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail);
|
||||
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
|
||||
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
|
||||
const innerLimit = offset + Math.max(limit * 5, 100);
|
||||
@@ -2176,7 +2148,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL
|
||||
THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject,
|
||||
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
|
||||
(${unverifiedExtractionFragment('p')}) AS unverified_stub,
|
||||
1 - (cc.${col} <=> ${castSql}) AS raw_score
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
@@ -2285,6 +2256,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.name === 'RetryAbortError') throw err;
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
if (isRetryableConnError(err)) {
|
||||
auditLogBatchExhausted(auditSite, batchSize, opts.maxRetries + 1, err);
|
||||
}
|
||||
@@ -2343,28 +2315,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
// Provenance fallback for chunks without an explicit `model`: resolve the
|
||||
// gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL.
|
||||
// #3461: getEmbeddingModel() THROWS when unconfigured (never returns
|
||||
// falsy) — on the throw path fall back to the brain's own
|
||||
// `config.embedding_model` row, then the compile-time default as the
|
||||
// last resort. See postgres-engine.ts _upsertChunksOnce for the full
|
||||
// rationale — pglite mirrors it for parity.
|
||||
let resolvedModel: string | null = null;
|
||||
// See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite
|
||||
// mirrors it for parity.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
} catch {
|
||||
try {
|
||||
const cfg = await this.db.query(
|
||||
`SELECT value FROM config WHERE key = 'embedding_model'`,
|
||||
);
|
||||
resolvedModel = ((cfg.rows[0] as { value?: string } | undefined)?.value) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2417,9 +2376,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch so the label always
|
||||
// describes whichever vector wins the upsert. See postgres-engine.ts for rationale.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2433,14 +2389,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
@@ -2901,11 +2850,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Remote MCP clients always land here.
|
||||
if (opts?.sourceIds && opts.sourceIds.length > 0) {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -2921,11 +2868,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// opts.sourceId, scope to that source (D20).
|
||||
if (opts?.sourceId) {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -2936,11 +2881,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -2957,11 +2900,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// foreign referrer nor a foreign origin slug is disclosed to the caller.
|
||||
if (opts?.sourceIds && opts.sourceIds.length > 0) {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -2974,11 +2915,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// v0.31.8 (D16) + #2200: federated arm above is first; two below mirror getLinks.
|
||||
if (opts?.sourceId) {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -2989,11 +2928,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3481,20 +3418,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return result;
|
||||
}
|
||||
|
||||
async getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>> {
|
||||
if (pageIds.length === 0) return new Set();
|
||||
// Parity with PostgresEngine.getUnverifiedExtractionPageIds (issue #160).
|
||||
// Predicate is the shared unverifiedExtractionFragment so this query and
|
||||
// the SQL-side source-boost guard can never drift.
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id FROM pages
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ${unverifiedExtractionFragment('pages')}`,
|
||||
[pageIds]
|
||||
);
|
||||
return new Set((rows as { id: number }[]).map((r) => Number(r.id)));
|
||||
}
|
||||
|
||||
async getPageTimestamps(slugs: string[]): Promise<Map<string, Date>> {
|
||||
if (slugs.length === 0) return new Map();
|
||||
const { rows } = await this.db.query(
|
||||
@@ -3876,6 +3799,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
async mergeOntologyFact(obs: OntologyObservationInput): Promise<OntologyMergeResult> {
|
||||
const sourceId = obs.sourceId ?? 'default';
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const dimension = normalizeDimension(obs.dimension);
|
||||
const vh = valueHash(obs.value);
|
||||
const conf = obs.confidence ?? 0.7;
|
||||
@@ -4309,11 +4233,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
$14, $15,
|
||||
$16, $17, $18, $19,
|
||||
$20
|
||||
)
|
||||
ON CONFLICT (source_id, source_markdown_slug, row_num)
|
||||
WHERE row_num IS NOT NULL
|
||||
DO NOTHING
|
||||
RETURNING id`
|
||||
) RETURNING id`
|
||||
: `INSERT INTO facts (
|
||||
source_id, entity_slug, fact, kind, visibility, notability, context,
|
||||
valid_from, valid_until, source, source_session, confidence,
|
||||
@@ -4327,16 +4247,12 @@ export class PGLiteEngine implements BrainEngine {
|
||||
$15, $16,
|
||||
$17, $18, $19, $20,
|
||||
$21
|
||||
)
|
||||
ON CONFLICT (source_id, source_markdown_slug, row_num)
|
||||
WHERE row_num IS NOT NULL
|
||||
DO NOTHING
|
||||
RETURNING id`,
|
||||
) RETURNING id`,
|
||||
embedStr === null
|
||||
? [ctx.source_id, entitySlug, input.fact, kind, visibility, notability, context, validFrom, validUntil, input.source, sourceSession, confidence, embeddedAt, input.row_num, input.source_markdown_slug, claimMetric, claimValue, claimUnit, claimPeriod, eventType]
|
||||
: [ctx.source_id, entitySlug, input.fact, kind, visibility, notability, context, validFrom, validUntil, input.source, sourceSession, confidence, embedStr, embeddedAt, input.row_num, input.source_markdown_slug, claimMetric, claimValue, claimUnit, claimPeriod, eventType],
|
||||
);
|
||||
if (ins.rows[0]) out.push(ins.rows[0].id);
|
||||
out.push(ins.rows[0].id);
|
||||
}
|
||||
return out;
|
||||
});
|
||||
@@ -4346,15 +4262,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
async deleteFactsForPage(
|
||||
slug: string,
|
||||
source_id: string,
|
||||
opts?: { excludeSourcePrefixes?: string[]; preserveExpiredLegacy?: boolean },
|
||||
opts?: { excludeSourcePrefixes?: string[] },
|
||||
): Promise<{ deleted: number }> {
|
||||
const prefixes = opts?.excludeSourcePrefixes;
|
||||
// #2646: keep soft-expired legacy rows (row_num NULL — never
|
||||
// fence-owned) so a fence reconcile can't destroy forget_fact's
|
||||
// legacy DB-only forget record.
|
||||
const expiredLegacyFilter = opts?.preserveExpiredLegacy
|
||||
? ` AND NOT (row_num IS NULL AND expired_at IS NOT NULL)`
|
||||
: '';
|
||||
if (prefixes && prefixes.length > 0) {
|
||||
// #1928: keep rows whose `source` matches an excluded prefix (e.g.
|
||||
// `cli:` conversation facts). COALESCE so NULL/empty-source fence rows
|
||||
@@ -4363,13 +4273,13 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const result = await this.db.query(
|
||||
`DELETE FROM facts
|
||||
WHERE source_id = $1 AND source_markdown_slug = $2
|
||||
AND NOT (COALESCE(source, '') LIKE ANY($3::text[]))${expiredLegacyFilter}`,
|
||||
AND NOT (COALESCE(source, '') LIKE ANY($3::text[]))`,
|
||||
[source_id, slug, patterns],
|
||||
);
|
||||
return { deleted: result.affectedRows ?? 0 };
|
||||
}
|
||||
const result = await this.db.query(
|
||||
`DELETE FROM facts WHERE source_id = $1 AND source_markdown_slug = $2${expiredLegacyFilter}`,
|
||||
`DELETE FROM facts WHERE source_id = $1 AND source_markdown_slug = $2`,
|
||||
[source_id, slug],
|
||||
);
|
||||
return { deleted: result.affectedRows ?? 0 };
|
||||
@@ -5373,16 +5283,12 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// pages_with_timeline) and v0.10.3 graph layer (link_coverage, timeline_coverage,
|
||||
// most_connected). Both coexist: master's brain_score is the composite
|
||||
// dashboard, v0.10.3 metrics give entity-page-level granularity.
|
||||
// #1305: every page-scoped count here excludes soft-deleted rows — same
|
||||
// posture as getStats — so brain_score moves when the user deletes pages.
|
||||
// Chunk/link counts stay raw (storage until the purge phase), matching
|
||||
// getStats, and destructive-removal counts elsewhere deliberately stay raw.
|
||||
const { rows: [h] } = await this.db.query(`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
@@ -5407,7 +5313,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL
|
||||
WHERE p.type IN ('entity', 'person', 'company')
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`);
|
||||
@@ -5426,7 +5332,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded,
|
||||
EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline
|
||||
FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
`);
|
||||
|
||||
const r = h as Record<string, unknown>;
|
||||
@@ -5521,18 +5426,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Sync
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number> {
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void> {
|
||||
newSlug = validateSlug(newSlug);
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// Source-qualify so a rename in source A doesn't sweep up same-slug rows
|
||||
// in sources B/C/D (mirrors postgres-engine.ts).
|
||||
const result = await this.db.query(
|
||||
await this.db.query(
|
||||
`UPDATE pages SET slug = $1, updated_at = now() WHERE slug = $2 AND source_id = $3`,
|
||||
[newSlug, oldSlug, sourceId]
|
||||
);
|
||||
// #3056: rows moved — a zero-row UPDATE does not throw, so the count is
|
||||
// the only way callers can see the no-op.
|
||||
return result.affectedRows ?? 0;
|
||||
}
|
||||
|
||||
async rewriteLinks(_oldSlug: string, _newSlug: string): Promise<void> {
|
||||
@@ -6046,6 +5948,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const recencyBias = opts.recency_bias ?? 'flat';
|
||||
let recencySql: string;
|
||||
if (recencyBias === 'on') {
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
recencySql = buildRecencyComponentSql({
|
||||
slugColumn: 'p.slug',
|
||||
dateExpr: 'COALESCE(p.effective_date, p.updated_at)',
|
||||
|
||||
+41
-139
@@ -13,29 +13,7 @@ import type {
|
||||
NewFact, FactListOpts, FactsHealth,
|
||||
SourceRow,
|
||||
} from './engine.ts';
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import { isConnectionEndedError } from './retry-matcher.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { logDbDisconnect } from './audit/db-disconnect-audit.ts';
|
||||
import { logPoolRecovery } from './audit/pool-recovery-audit.ts';
|
||||
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
|
||||
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
|
||||
import type {
|
||||
DomainBankSampleOpts, CorpusSampleOpts, DomainBankRow,
|
||||
@@ -87,7 +65,6 @@ import { logConnectionEvent } from './connection-audit.ts';
|
||||
import { validateSlug, contentHash, rowToPage, rowToStalePage, rowToChunk, rowToSearchResult, parseEmbedding, tryParseEmbedding, takeRowToTake, takeHitRowToHit, isUndefinedTableError, warnOncePerProcess } from './utils.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from './extraction-review.ts';
|
||||
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts';
|
||||
import { DELETE_BATCH_SIZE } from './engine-constants.ts';
|
||||
import { SOURCE_CONFIG_OBJECT_SQL } from './source-config-sql.ts';
|
||||
@@ -353,6 +330,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// even a no-op disconnect (engine that was never connected) is
|
||||
// recorded — that case may itself be a caller-side bug worth seeing.
|
||||
try {
|
||||
const { logDbDisconnect } = await import('./audit/db-disconnect-audit.ts');
|
||||
logDbDisconnect('postgres', this._connectionStyle ?? 'unknown');
|
||||
} catch { /* best-effort; never block disconnect on audit failure */ }
|
||||
// v0.30.1: tear down the direct pool first if the manager owns one.
|
||||
@@ -402,13 +380,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
let dims: number = DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
|
||||
const sqlText = getPostgresSchema(dims, model);
|
||||
@@ -1056,8 +1030,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
const rows = await tx`
|
||||
SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
FROM pages
|
||||
WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
|
||||
LIMIT 1
|
||||
@@ -2147,10 +2120,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// innerLimit scales with offset to preserve the pagination contract:
|
||||
// a fixed cap of 100 would silently empty offset > 100.
|
||||
const boostMap = resolveBoostMap();
|
||||
// issue #160: the guard predicate is projected as `unverified_stub` in
|
||||
// hnsw_candidates (frontmatter isn't otherwise available at re-rank), so
|
||||
// unverified auto-extracted stubs get factor 1.0, not the people/ 1.2x.
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail, 'unverified_stub');
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail);
|
||||
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
|
||||
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
|
||||
const innerLimit = offset + Math.max(limit * 5, 100);
|
||||
@@ -2250,7 +2220,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL
|
||||
THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject,
|
||||
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
|
||||
(${unverifiedExtractionFragment('p')}) AS unverified_stub,
|
||||
1 - (cc.${col} <=> ${castSql}) AS raw_score
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
@@ -2404,8 +2373,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
if (err instanceof Error && err.name === 'RetryAbortError') throw err;
|
||||
// Best-effort exhausted-retry log. If the error wasn't retryable in
|
||||
// the first place, isRetryableConnError(err) is false and we skip.
|
||||
// retry.ts is already in this module's static graph through withRetry, so
|
||||
// classifying the exhausted error does not need a second runtime import.
|
||||
// Lazy-import to avoid a circular dep concern.
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
if (isRetryableConnError(err)) {
|
||||
auditLogBatchExhausted(auditSite, batchSize, opts.maxRetries + 1, err);
|
||||
}
|
||||
@@ -2463,30 +2432,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
// hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors
|
||||
// were produced by a different, config-resolved model — corrupting the
|
||||
// provenance that signature-drift staleness + dim-migration logic trust.
|
||||
//
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured —
|
||||
// it never returns falsy — so an `||` guard here is dead code and the
|
||||
// catch path used to stamp the compile-time default onto rows whose
|
||||
// vectors came from the config-resolved provider. On the throw path we
|
||||
// now fall back to the brain's own `config.embedding_model` row (kept
|
||||
// current by init / migrate / retrieval-upgrade), which names the model
|
||||
// that actually produced this brain's vectors. The compile-time default
|
||||
// is the LAST resort (fresh brain whose config row doesn't exist yet).
|
||||
let resolvedModel: string | null = null;
|
||||
// Mirrors the resolve-then-fallback pattern used for schema sizing above.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
} catch {
|
||||
try {
|
||||
const cfg = await sql`SELECT value FROM config WHERE key = 'embedding_model'`;
|
||||
resolvedModel = (cfg[0]?.value as string | undefined) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2550,11 +2503,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch — the label must
|
||||
// describe whichever vector WINS the upsert. The old COALESCE(EXCLUDED.model, …)
|
||||
// relabeled preserved (older-model) vectors with the current gateway model on every
|
||||
// partial re-embed, corrupting provenance without changing the vector.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2568,14 +2516,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
@@ -3052,11 +2993,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
if (opts?.sourceIds && opts.sourceIds.length > 0) {
|
||||
const ids = opts.sourceIds;
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3071,11 +3010,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
// opts.sourceId, scope the from-page lookup.
|
||||
if (opts?.sourceId) {
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3085,11 +3022,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3111,11 +3046,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
if (opts?.sourceIds && opts.sourceIds.length > 0) {
|
||||
const ids = opts.sourceIds;
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3127,11 +3060,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
// v0.31.8 (D16) + #2200: federated arm above is first; two below mirror getLinks.
|
||||
if (opts?.sourceId) {
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3141,11 +3072,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
const rows = await tx`
|
||||
SELECT f.slug as from_slug, f.source_id as from_source_id,
|
||||
t.slug as to_slug, t.source_id as to_source_id,
|
||||
SELECT f.slug as from_slug, t.slug as to_slug,
|
||||
l.link_type, l.context, l.link_source,
|
||||
o.slug as origin_slug, o.source_id as origin_source_id,
|
||||
l.origin_field
|
||||
o.slug as origin_slug, l.origin_field
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
@@ -3642,20 +3571,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
return result;
|
||||
}
|
||||
|
||||
async getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>> {
|
||||
if (pageIds.length === 0) return new Set();
|
||||
const sql = this.sql;
|
||||
// Predicate is the shared unverifiedExtractionFragment (issue #160) so
|
||||
// this query and the SQL-side source-boost guard can never drift.
|
||||
const rows = await sql.unsafe(
|
||||
`SELECT id FROM pages
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ${unverifiedExtractionFragment('pages')}`,
|
||||
[pageIds] as never,
|
||||
);
|
||||
return new Set((rows as unknown as { id: number }[]).map((r) => Number(r.id)));
|
||||
}
|
||||
|
||||
async getPageTimestamps(slugs: string[]): Promise<Map<string, Date>> {
|
||||
if (slugs.length === 0) return new Map();
|
||||
const sql = this.sql;
|
||||
@@ -4020,6 +3935,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
async mergeOntologyFact(obs: OntologyObservationInput): Promise<OntologyMergeResult> {
|
||||
const sql = this.sql;
|
||||
const sourceId = obs.sourceId ?? 'default';
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const dimension = normalizeDimension(obs.dimension);
|
||||
const vh = valueHash(obs.value);
|
||||
const conf = obs.confidence ?? 0.7;
|
||||
@@ -4491,13 +4407,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
${input.row_num}, ${input.source_markdown_slug},
|
||||
${claimMetric}, ${claimValue}, ${claimUnit}, ${claimPeriod},
|
||||
${eventType}
|
||||
)
|
||||
ON CONFLICT (source_id, source_markdown_slug, row_num)
|
||||
WHERE row_num IS NOT NULL
|
||||
DO NOTHING
|
||||
RETURNING id
|
||||
) RETURNING id
|
||||
`;
|
||||
if (ins[0]) out.push(Number(ins[0].id));
|
||||
out.push(Number(ins[0].id));
|
||||
}
|
||||
return out;
|
||||
});
|
||||
@@ -4507,16 +4419,10 @@ export class PostgresEngine implements BrainEngine {
|
||||
async deleteFactsForPage(
|
||||
slug: string,
|
||||
source_id: string,
|
||||
opts?: { excludeSourcePrefixes?: string[]; preserveExpiredLegacy?: boolean },
|
||||
opts?: { excludeSourcePrefixes?: string[] },
|
||||
): Promise<{ deleted: number }> {
|
||||
const sql = this.sql;
|
||||
const prefixes = opts?.excludeSourcePrefixes;
|
||||
// #2646: keep soft-expired legacy rows (row_num NULL — never
|
||||
// fence-owned) so a fence reconcile can't destroy forget_fact's
|
||||
// legacy DB-only forget record.
|
||||
const expiredLegacyFilter = opts?.preserveExpiredLegacy
|
||||
? sql`AND NOT (row_num IS NULL AND expired_at IS NOT NULL)`
|
||||
: sql``;
|
||||
if (prefixes && prefixes.length > 0) {
|
||||
// #1928: keep rows whose `source` matches an excluded prefix (e.g.
|
||||
// `cli:` conversation facts). COALESCE so NULL/empty-source fence rows
|
||||
@@ -4527,12 +4433,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
WHERE source_id = ${source_id}
|
||||
AND source_markdown_slug = ${slug}
|
||||
AND NOT (COALESCE(source, '') LIKE ANY(${patterns}))
|
||||
${expiredLegacyFilter}
|
||||
`;
|
||||
return { deleted: result.count ?? 0 };
|
||||
}
|
||||
const result = await sql`
|
||||
DELETE FROM facts WHERE source_id = ${source_id} AND source_markdown_slug = ${slug} ${expiredLegacyFilter}
|
||||
DELETE FROM facts WHERE source_id = ${source_id} AND source_markdown_slug = ${slug}
|
||||
`;
|
||||
return { deleted: result.count ?? 0 };
|
||||
}
|
||||
@@ -5472,16 +5377,12 @@ export class PostgresEngine implements BrainEngine {
|
||||
// no outbound links). The raw islanded list is filtered through the same
|
||||
// policy as `gbrain orphans` so convention pages do not count against
|
||||
// dashboard health.
|
||||
// #1305: every page-scoped count here excludes soft-deleted rows — same
|
||||
// posture as getStats — so brain_score moves when the user deletes pages.
|
||||
// Chunk/link counts stay raw (storage until the purge phase), matching
|
||||
// getStats, and destructive-removal counts elsewhere deliberately stay raw.
|
||||
const [h] = await sql`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL
|
||||
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
@@ -5503,7 +5404,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL
|
||||
WHERE p.type IN ('entity', 'person', 'company')
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`;
|
||||
@@ -5522,7 +5423,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded,
|
||||
EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline
|
||||
FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const pageCount = Number(h.page_count);
|
||||
@@ -5614,17 +5514,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Sync
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<number> {
|
||||
async updateSlug(oldSlug: string, newSlug: string, opts?: { sourceId?: string }): Promise<void> {
|
||||
newSlug = validateSlug(newSlug);
|
||||
const sql = this.sql;
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// Source-qualify so a rename in source A doesn't sweep up same-slug rows
|
||||
// in sources B/C/D (which would either rename them all OR fail the
|
||||
// (source_id, slug) UNIQUE if the new slug already exists in another source).
|
||||
const result = await sql`UPDATE pages SET slug = ${newSlug}, updated_at = now() WHERE slug = ${oldSlug} AND source_id = ${sourceId}`;
|
||||
// #3056: rows moved — a zero-row UPDATE does not throw, so the count is
|
||||
// the only way callers can see the no-op.
|
||||
return result.count ?? 0;
|
||||
await sql`UPDATE pages SET slug = ${newSlug}, updated_at = now() WHERE slug = ${oldSlug} AND source_id = ${sourceId}`;
|
||||
}
|
||||
|
||||
async rewriteLinks(_oldSlug: string, _newSlug: string): Promise<void> {
|
||||
@@ -5863,10 +5760,12 @@ export class PostgresEngine implements BrainEngine {
|
||||
let isReap = false;
|
||||
if (ctx?.error !== undefined) {
|
||||
try {
|
||||
const { isConnectionEndedError } = await import('./retry-matcher.ts');
|
||||
isReap = isConnectionEndedError(ctx.error);
|
||||
} catch { /* classification is best-effort */ }
|
||||
}
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery(isReap ? 'reap_detected' : 'reconnect_other', ctx?.error);
|
||||
} catch { /* audit is best-effort */ }
|
||||
|
||||
@@ -5890,6 +5789,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// New pool is live — discard the old one best-effort.
|
||||
if (oldSql) { try { await oldSql.end({ timeout: 5 }); } catch { /* swallow */ } }
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery('reconnect_succeeded');
|
||||
} catch { /* best-effort */ }
|
||||
} catch (err) {
|
||||
@@ -5901,6 +5801,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
this._sql = oldSql;
|
||||
this.connectionManager = oldManager;
|
||||
try {
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
logPoolRecovery('reconnect_failed', err);
|
||||
} catch { /* best-effort */ }
|
||||
throw err; // let batchRetry's backoff handle the retry
|
||||
@@ -6337,6 +6238,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
const recencyBias = opts.recency_bias ?? 'flat';
|
||||
let recencySql: string;
|
||||
if (recencyBias === 'on') {
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
recencySql = buildRecencyComponentSql({
|
||||
slugColumn: 'p.slug',
|
||||
dateExpr: 'COALESCE(p.effective_date, p.updated_at)',
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user