mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-17 10:22:34 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3487e4b255 | ||
|
|
159ddc3249 |
@@ -71,8 +71,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
|
||||
@@ -131,9 +131,7 @@ into gbrain so other clients can scaffold it. Default behavior:
|
||||
`~/.gbrain/harvest-private-patterns.txt` plus built-in defaults
|
||||
(canonical private fork name, common email regex, Slack channel pattern). Any
|
||||
match → rollback (delete the harvested files) and exit non-zero.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted. Harvest must preserve
|
||||
the top-level OpenClaw-native plugin fields (`id`, `configSchema`, `contracts`)
|
||||
because OpenClaw validates those before it can install the package.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted.
|
||||
- `--no-lint` bypasses the linter (after a manual editorial scrub).
|
||||
|
||||
Use the `skillpack-harvest` skill (its companion editorial workflow)
|
||||
|
||||
@@ -233,14 +233,13 @@ keep it or `git checkout` to throw it away. Nothing is committed for you.
|
||||
|
||||
**For a skill that ships with gbrain** (anything under the gbrain repo's own
|
||||
`skills/`): SkillOpt refuses to overwrite it by default and writes the winner to
|
||||
`skills/<name>/skillopt/proposed.md` instead (while keeping `best.md` as the
|
||||
optimizer's current-best pointer), so an optimization pass can never silently
|
||||
mutate a skill other people depend on. Two ways to handle that:
|
||||
`skills/<name>/skillopt/best.md` instead, so an optimization pass can never
|
||||
silently mutate a skill other people depend on. Two ways to handle that:
|
||||
|
||||
```bash
|
||||
# See the proposed improvement without touching SKILL.md (works for ANY skill):
|
||||
gbrain skillopt meeting-prep --split 1:1:1 --no-mutate
|
||||
# → writes skills/meeting-prep/skillopt/proposed.md, updates best.md, and prints the proposal path.
|
||||
# → writes skills/meeting-prep/skillopt/best.md (the proposed rewrite), prints its path. Copy what you want.
|
||||
|
||||
# Actually rewrite a bundled skill (explicit opt-in + an independent held-out set):
|
||||
gbrain skillopt brain-ops --split 1:1:1 --allow-mutate-bundled \
|
||||
|
||||
+2
-2
@@ -1565,8 +1565,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.32.3.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
|
||||
@@ -23,7 +23,7 @@ health_checks:
|
||||
label: "Auth provider"
|
||||
checks:
|
||||
- type: http
|
||||
url: "$CLAWVISOR_URL/ready"
|
||||
url: "$CLAWVISOR_URL/health"
|
||||
label: "ClawVisor"
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
@@ -135,7 +135,7 @@ Tell the user:
|
||||
|
||||
Validate:
|
||||
```bash
|
||||
curl -sf "$CLAWVISOR_URL/ready" && echo "PASS: ClawVisor reachable" || echo "FAIL"
|
||||
curl -sf "$CLAWVISOR_URL/health" && echo "PASS: ClawVisor reachable" || echo "FAIL"
|
||||
```
|
||||
|
||||
**STOP until ClawVisor validates.**
|
||||
|
||||
@@ -1,308 +0,0 @@
|
||||
---
|
||||
id: contacts-to-brain
|
||||
name: Contacts-to-Brain
|
||||
version: 0.1.0
|
||||
description: Google Contacts become canonical people/ pages, enriching brain entities with ground-truth name/email/phone/org data.
|
||||
category: sense
|
||||
requires: [credential-gateway]
|
||||
secrets:
|
||||
- name: CLAWVISOR_URL
|
||||
description: ClawVisor gateway URL (Option A — recommended, handles OAuth for you)
|
||||
where: https://clawvisor.com — create an agent, activate Google Contacts service
|
||||
- name: CLAWVISOR_AGENT_TOKEN
|
||||
description: ClawVisor agent token (Option A)
|
||||
where: https://clawvisor.com — agent settings, copy the agent token
|
||||
- name: GOOGLE_CLIENT_ID
|
||||
description: Google OAuth2 client ID (Option B — direct API access, you manage tokens)
|
||||
where: https://console.cloud.google.com/apis/credentials — create OAuth 2.0 Client ID
|
||||
- name: GOOGLE_CLIENT_SECRET
|
||||
description: Google OAuth2 client secret (Option B)
|
||||
where: https://console.cloud.google.com/apis/credentials — same page as client ID
|
||||
health_checks:
|
||||
- type: any_of
|
||||
label: "Auth provider"
|
||||
checks:
|
||||
- type: http
|
||||
url: "$CLAWVISOR_URL/ready"
|
||||
label: "ClawVisor"
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
label: "Google OAuth"
|
||||
setup_time: 15 min
|
||||
cost_estimate: "$0 (both options are free)"
|
||||
---
|
||||
|
||||
# Contacts-to-Brain: Your Address Book Becomes Ground Truth
|
||||
|
||||
Calendar attendees and email senders are the bulk of `people/<slug>` brain pages.
|
||||
Your Google Contacts is the ground-truth directory — canonical name, email, phone,
|
||||
organization — for those same entities. Syncing it closes the "who is this person"
|
||||
loop automatically and feeds enrichment.
|
||||
|
||||
## IMPORTANT: Instructions for the Agent
|
||||
|
||||
**You are the installer.** Follow these steps precisely.
|
||||
|
||||
**Why this matters:** email-to-brain and calendar-to-brain create people pages
|
||||
from whatever name string the API returned — sometimes an email prefix, sometimes
|
||||
a nickname. Contacts carries the authoritative record. After this recipe runs,
|
||||
"j.smith@acme-example.com" and "Jon S" resolve to the same person page with the
|
||||
right display name, phone, and company.
|
||||
|
||||
**The output is staging files, not direct writes:** the deterministic collector
|
||||
dumps contact records to `brain/contacts/.staging/`; YOU (the agent) merge them
|
||||
into `people/<slug>` pages using judgment — the Notability Gate in
|
||||
`skills/_brain-filing-rules.md` applies. Not every contact deserves a page.
|
||||
|
||||
**Do not skip steps. Verify after each step.**
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Google Contacts (People API, paginated)
|
||||
↓ (ClawVisor credential gateway: list_contacts / get_contact / search_contacts)
|
||||
Contacts Sync Script (deterministic Node.js)
|
||||
↓ Outputs:
|
||||
├── brain/contacts/.raw/contacts-{date}.json (raw API responses, provenance)
|
||||
└── brain/contacts/.staging/{slug}.md (one markdown record per contact)
|
||||
↓
|
||||
Agent reads staging files
|
||||
↓ Judgment calls (Notability Gate):
|
||||
├── Merge into existing people/<slug> pages (name/email/phone/org enrichment)
|
||||
├── Create new pages ONLY for notable contacts not already in brain
|
||||
└── Skip the rest (staging is not the brain)
|
||||
```
|
||||
|
||||
## Opinionated Defaults
|
||||
|
||||
**Staging record format** (one file per contact, deterministic):
|
||||
```markdown
|
||||
# Alice Example
|
||||
|
||||
- **Emails:** alice@acme-example.com, alice@gmail.com
|
||||
- **Phone:** +1 555 0100
|
||||
- **Organization:** Acme Example — VP Engineering
|
||||
- **Source:** Google Contacts (resourceName people/c123, synced 2026-07-21)
|
||||
```
|
||||
|
||||
**Enrichment, not duplication:** if `people/alice-example.md` already exists,
|
||||
append missing fields to it with a `[Source: Google Contacts]` citation. Do NOT
|
||||
create a second page. Slug-match by normalized name, then by email against
|
||||
existing page content.
|
||||
|
||||
**Notability Gate (from `skills/_brain-filing-rules.md`):** a contact with no
|
||||
brain presence gets a new page only if they appear elsewhere in the brain
|
||||
(calendar attendee, email correspondent) or the user confirms they matter.
|
||||
When in doubt, DON'T create — a junk page degrades search quality.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **GBrain installed and configured** (`gbrain doctor` passes)
|
||||
2. **Node.js 18+** (for the sync script)
|
||||
3. **Google Contacts access** via ONE of:
|
||||
- **Option A: ClawVisor** (recommended, handles OAuth for you, no token management)
|
||||
- **Option B: Google OAuth2 directly** (you manage tokens, no extra service needed)
|
||||
|
||||
## Setup Flow
|
||||
|
||||
### Step 1: Choose and Configure Contacts Access
|
||||
|
||||
Ask the user: "How do you want to connect to Google Contacts?
|
||||
|
||||
**Option A: ClawVisor (recommended)**
|
||||
ClawVisor handles OAuth, token refresh, and encryption. If you already use
|
||||
ClawVisor for email-to-brain or calendar-to-brain, this uses the same setup —
|
||||
just activate the Google Contacts service on your existing agent.
|
||||
|
||||
**Option B: Google OAuth2 directly**
|
||||
Connect to the Google People API directly. No extra service needed, but you
|
||||
manage OAuth tokens yourself."
|
||||
|
||||
#### Option A: ClawVisor Setup
|
||||
|
||||
Tell the user:
|
||||
"I need your ClawVisor URL and agent token.
|
||||
1. Go to https://clawvisor.com
|
||||
2. Create an agent (or use existing)
|
||||
3. Activate the **Google Contacts** service
|
||||
4. Create a standing task with purpose: 'Full contacts access for people
|
||||
enrichment: list contacts, read contact details, search contacts across
|
||||
all connected Google accounts.'
|
||||
IMPORTANT: Be EXPANSIVE in the task purpose. Narrow purposes block requests.
|
||||
5. Copy the gateway URL and agent token"
|
||||
|
||||
Validate:
|
||||
```bash
|
||||
curl -sf "$CLAWVISOR_URL/ready" && echo "PASS: ClawVisor reachable" || echo "FAIL"
|
||||
```
|
||||
|
||||
**STOP until ClawVisor validates.**
|
||||
|
||||
#### Option B: Google OAuth2 Setup
|
||||
|
||||
Same flow as `recipes/credential-gateway.md` Option B, with the contacts scope:
|
||||
|
||||
1. https://console.cloud.google.com/apis/credentials — create an OAuth client ID
|
||||
(Desktop app), consent screen scope: `https://www.googleapis.com/auth/contacts.readonly`
|
||||
2. Enable the People API: https://console.cloud.google.com/apis/library/people.googleapis.com
|
||||
3. Run the OAuth flow; store tokens in `~/.gbrain/google-tokens.json` (auto-refresh on expiry)
|
||||
|
||||
Validate:
|
||||
```bash
|
||||
[ -n "$GOOGLE_CLIENT_ID" ] && [ -n "$GOOGLE_CLIENT_SECRET" ] \
|
||||
&& echo "PASS: Google OAuth credentials set" \
|
||||
|| echo "FAIL: Missing GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET"
|
||||
```
|
||||
|
||||
**STOP until OAuth flow completes and tokens are stored.**
|
||||
|
||||
### Step 2: Set Up the Contacts Sync Script
|
||||
|
||||
```bash
|
||||
mkdir -p contacts-sync
|
||||
cd contacts-sync
|
||||
npm init -y
|
||||
```
|
||||
|
||||
The sync script needs these capabilities:
|
||||
|
||||
1. **Paginated retrieval** — `list_contacts` (People API `people.connections.list`)
|
||||
returns pages of up to 100; follow `nextPageToken` until exhausted. Request
|
||||
fields: names, emailAddresses, phoneNumbers, organizations, metadata.
|
||||
2. **Deterministic staging output** — one markdown file per contact at
|
||||
`brain/contacts/.staging/{slug}.md`, slug from normalized display name
|
||||
(fall back to email prefix). Same contact = same file on every run (idempotent).
|
||||
3. **Raw JSON preservation** — save raw API responses to
|
||||
`brain/contacts/.raw/contacts-{date}.json` for provenance.
|
||||
4. **Skip empty records** — contacts with no name AND no email are noise; drop them.
|
||||
|
||||
### Step 3: Run the Full Sync
|
||||
|
||||
```bash
|
||||
node contacts-sync.mjs
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
ls brain/contacts/.staging/ | head -10
|
||||
```
|
||||
|
||||
Should show one file per contact, e.g. `alice-example.md`, `charlie-example.md`.
|
||||
|
||||
### Step 4: Enrich People Pages (Agent Judgment)
|
||||
|
||||
This is YOUR job (the agent). For each staging record:
|
||||
|
||||
1. **Check brain**: `gbrain search "contact name"` — do they have a
|
||||
`people/<slug>` page? Also search by email address.
|
||||
2. **Existing page** → merge the ground-truth fields (canonical name, emails,
|
||||
phone, organization) into the page, each with a
|
||||
`[Source: Google Contacts]` citation. Fix a wrong/partial display name.
|
||||
3. **No page** → apply the Notability Gate: create a page only if the contact
|
||||
already appears in the brain (calendar, email) or is clearly relevant.
|
||||
Otherwise skip.
|
||||
4. **Back-link** per the Iron Law in `skills/_brain-filing-rules.md`: an
|
||||
organization with a brain page gets a link from the person's page and back.
|
||||
|
||||
After enrichment, import and embed:
|
||||
```bash
|
||||
gbrain sync --no-pull --no-embed && gbrain embed --stale
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
gbrain search "alice-example" --limit 3
|
||||
```
|
||||
|
||||
Should return the enriched people page with contact details.
|
||||
|
||||
### Step 5: Set Up Weekly Sync
|
||||
|
||||
Contacts change slowly; once a week is plenty:
|
||||
```bash
|
||||
# Cron: every Sunday at 9 AM
|
||||
0 9 * * 0 cd /path/to/contacts-sync && node contacts-sync.mjs
|
||||
```
|
||||
|
||||
After each sync, re-run the Step 4 enrichment pass over CHANGED staging files
|
||||
only (compare mtime or diff against git), then:
|
||||
```bash
|
||||
gbrain sync --no-pull --no-embed && gbrain embed --stale
|
||||
```
|
||||
|
||||
### Step 6: Log Setup Completion
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/contacts-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.1.0","status":"ok","details":{"contacts":"CONTACT_COUNT"}}' >> ~/.gbrain/integrations/contacts-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
Tell the user: "Contacts-to-brain is set up. [N] contacts staged; [M] people
|
||||
pages enriched with ground-truth contact data. Weekly sync keeps it current."
|
||||
|
||||
## Implementation Guide
|
||||
|
||||
### Pagination
|
||||
|
||||
```
|
||||
list_all_contacts():
|
||||
contacts = []
|
||||
token = null
|
||||
do:
|
||||
page = list_contacts({ pageSize: 100, pageToken: token,
|
||||
personFields: 'names,emailAddresses,phoneNumbers,organizations,metadata' })
|
||||
contacts += page.connections
|
||||
token = page.nextPageToken
|
||||
while token
|
||||
return contacts
|
||||
```
|
||||
|
||||
### Slug Normalization
|
||||
|
||||
```
|
||||
slugify(contact):
|
||||
name = contact.names?[0]?.displayName
|
||||
if not name:
|
||||
name = contact.emailAddresses?[0]?.value.split('@')[0]
|
||||
return name.toLowerCase().normalize('NFD')
|
||||
.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
||||
```
|
||||
|
||||
Collision (two contacts, same slug): suffix with the email domain
|
||||
(`alice-example-acme-example`) rather than overwrite.
|
||||
|
||||
### What the Agent Should Test After Setup
|
||||
|
||||
1. **Idempotency:** run the sync twice. `git status` on `brain/contacts/.staging/`
|
||||
shows no changes on the second run.
|
||||
2. **Pagination:** with 250+ contacts, verify the staging count matches the
|
||||
Google Contacts count (not capped at 100).
|
||||
3. **Notability Gate:** verify a one-off contact with no brain presence did NOT
|
||||
get a `people/` page.
|
||||
4. **Enrichment merge:** verify an existing people page gained contact fields
|
||||
without losing its prior content, each with a `[Source: Google Contacts]`
|
||||
citation.
|
||||
|
||||
## Cost Estimate
|
||||
|
||||
| Component | Monthly Cost |
|
||||
|-----------|-------------|
|
||||
| ClawVisor (free tier) | $0 |
|
||||
| Google People API | $0 (within free quota) |
|
||||
| **Total** | **$0** |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**No contacts returned:**
|
||||
- Check ClawVisor has the Google Contacts service activated
|
||||
- Check the standing task purpose is expansive enough
|
||||
- Option B: verify the People API is enabled and the token carries
|
||||
`contacts.readonly`
|
||||
|
||||
**Duplicate people pages after enrichment:**
|
||||
- The agent matched by name but the brain page slug differs — search by email
|
||||
address too before creating, then merge and delete the duplicate
|
||||
|
||||
**Contacts with no name:**
|
||||
- The sync script falls back to the email prefix; records with neither name
|
||||
nor email are dropped as noise
|
||||
@@ -23,7 +23,7 @@ health_checks:
|
||||
label: "Auth provider"
|
||||
checks:
|
||||
- type: http
|
||||
url: "$CLAWVISOR_URL/ready"
|
||||
url: "$CLAWVISOR_URL/health"
|
||||
label: "ClawVisor"
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
@@ -75,7 +75,7 @@ Tell the user:
|
||||
3. Activate the services you need:
|
||||
- **Gmail** (for email-to-brain)
|
||||
- **Google Calendar** (for calendar-to-brain)
|
||||
- **Google Contacts** (for contacts-to-brain)
|
||||
- **Google Contacts** (for enrichment)
|
||||
4. Create a standing task with a broad purpose. CRITICAL: be EXPANSIVE.
|
||||
|
||||
Good purpose: 'Full executive assistant access to Gmail, Calendar, and
|
||||
@@ -88,7 +88,7 @@ Tell the user:
|
||||
|
||||
Validate:
|
||||
```bash
|
||||
curl -sf "$CLAWVISOR_URL/ready" \
|
||||
curl -sf "$CLAWVISOR_URL/health" \
|
||||
&& echo "PASS: ClawVisor reachable" \
|
||||
|| echo "FAIL: ClawVisor not reachable — check the URL"
|
||||
```
|
||||
@@ -171,7 +171,7 @@ can now access your Google services."
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **ClawVisor:** `curl $CLAWVISOR_URL/ready` returns OK.
|
||||
1. **ClawVisor:** `curl $CLAWVISOR_URL/health` returns OK.
|
||||
2. **Google OAuth:** Tokens exist at `~/.gbrain/google-tokens.json`.
|
||||
3. **Gmail access:** Run the email collector — it should pull recent messages.
|
||||
4. **Calendar access:** Run the calendar sync — it should pull today's events.
|
||||
|
||||
@@ -23,7 +23,7 @@ health_checks:
|
||||
label: "Auth provider"
|
||||
checks:
|
||||
- type: http
|
||||
url: "$CLAWVISOR_URL/ready"
|
||||
url: "$CLAWVISOR_URL/health"
|
||||
label: "ClawVisor"
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
@@ -130,7 +130,7 @@ Tell the user:
|
||||
|
||||
Validate:
|
||||
```bash
|
||||
curl -sf "$CLAWVISOR_URL/ready" && echo "PASS: ClawVisor reachable" || echo "FAIL"
|
||||
curl -sf "$CLAWVISOR_URL/health" && echo "PASS: ClawVisor reachable" || echo "FAIL"
|
||||
```
|
||||
|
||||
**STOP until ClawVisor validates.**
|
||||
@@ -328,7 +328,7 @@ threads you already replied to. Sent mail acts as a negative filter.
|
||||
## Troubleshooting
|
||||
|
||||
**No emails collected:**
|
||||
- Check ClawVisor health: `curl $CLAWVISOR_URL/ready`
|
||||
- Check ClawVisor health: `curl $CLAWVISOR_URL/health`
|
||||
- Check standing task is active and has Gmail service enabled
|
||||
- Check task purpose is expansive enough (narrow purposes block requests)
|
||||
|
||||
|
||||
@@ -266,5 +266,4 @@ editorial pass.
|
||||
(e.g. `src/commands/<slug>.ts` if the host SKILL.md declares it
|
||||
in frontmatter)
|
||||
- gbrain's `openclaw.plugin.json` — adds the slug to `skills:`
|
||||
array, sorted alphabetically, without removing OpenClaw-native plugin fields
|
||||
like `id`, `configSchema`, or `contracts`
|
||||
array, sorted alphabetically
|
||||
|
||||
@@ -57,8 +57,6 @@ This mode guarantees:
|
||||
- `skills/manifest.json` lists every skill directory
|
||||
- `skills/RESOLVER.md` references every skill in the manifest
|
||||
- `openclaw.plugin.json` `skills[]` round-trips with both
|
||||
- `openclaw.plugin.json` keeps OpenClaw install-required native plugin fields
|
||||
(`id`, object `configSchema`, and `contracts.contextEngines` when applicable)
|
||||
- No MECE violations (duplicate triggers across skills)
|
||||
|
||||
### Phases
|
||||
@@ -74,7 +72,7 @@ This mode guarantees:
|
||||
### Automation
|
||||
|
||||
```bash
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts test/openclaw-plugin-manifest.test.ts
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts
|
||||
```
|
||||
|
||||
The CI-gated check is the package.json `test` script.
|
||||
|
||||
+1
-8
@@ -54,7 +54,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -78,8 +78,6 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
'capture',
|
||||
// v0.42 self-upgrade ships its own usage (flags + the agent-skill story).
|
||||
'self-upgrade',
|
||||
// maintain (#3015) prints its own usage block (modes + not-auto-applied list).
|
||||
'maintain',
|
||||
// v0.43 (#2095): watch ships WATCH_HELP (flags + the stdin-turn protocol).
|
||||
'watch',
|
||||
// v0.37 fix wave (Lane D.4 + CDX2-12): sync's --no-embed flag was
|
||||
@@ -1759,11 +1757,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runOrphans(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'maintain': {
|
||||
const { runMaintain } = await import('./commands/maintain.ts');
|
||||
await runMaintain(engine, args);
|
||||
break;
|
||||
}
|
||||
// v0.32.7 CJK wave — post-upgrade markdown re-chunk sweep.
|
||||
// v0.36 Phase 3 wave — `gbrain reindex --multimodal` re-embeds content_chunks
|
||||
// into the unified Voyage multimodal-3 column.
|
||||
|
||||
+47
-47
@@ -1,6 +1,7 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { embedBatch, currentEmbeddingSignature } from '../core/embedding.ts';
|
||||
import type { ChunkInput } from '../core/types.ts';
|
||||
import type { ChunkInput, ResolvedColumn } from '../core/types.ts';
|
||||
import { resolveWriteColumnForEngine } from '../core/search/embedding-column.ts';
|
||||
import { chunkText } from '../core/chunkers/recursive.ts';
|
||||
import { createProgress, type ProgressReporter } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
@@ -183,8 +184,13 @@ export class EmbeddingDimMismatchError extends Error {
|
||||
* fresh-install bug class at the very first invocation instead of letting
|
||||
* the worker pool hammer N pages with raw 22000 errors.
|
||||
*/
|
||||
async function preflightDimMismatch(engine: BrainEngine, dryRun: boolean): Promise<void> {
|
||||
async function preflightDimMismatch(engine: BrainEngine, dryRun: boolean, embeddingColumn?: ResolvedColumn): Promise<void> {
|
||||
if (dryRun) return; // dry-run never embeds, no risk
|
||||
// #1262: an alt-column brain writes to `embeddingColumn`, not the legacy
|
||||
// `embedding` column — the legacy column's dims are irrelevant, and the
|
||||
// registry entry (validated at resolve time) pins the target's dims. Only
|
||||
// the legacy default path needs the schema-vs-gateway dim comparison.
|
||||
if (embeddingColumn && embeddingColumn.name !== 'embedding') return;
|
||||
const { readContentChunksEmbeddingDim, embeddingMismatchMessage } = await import('../core/embedding-dim-check.ts');
|
||||
const { getEmbeddingDimensions, getEmbeddingModel } = await import('../core/ai/gateway.ts');
|
||||
let existing;
|
||||
@@ -238,7 +244,12 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
// v0.37.11.0 (Lane D.2): pre-flight dim-mismatch check. Catches the headline
|
||||
// fresh-install bug class before the worker pool spends 20 parallel calls
|
||||
// hitting raw Postgres dimension errors.
|
||||
await preflightDimMismatch(engine, !!opts.dryRun);
|
||||
// #1262: resolve the write-side embedding column ONCE at the boundary
|
||||
// (merged config + gateway model) and thread the descriptor through every
|
||||
// upsertChunks / stale-scan below. undefined => legacy `embedding` column.
|
||||
const embeddingColumn = await resolveWriteColumnForEngine(engine);
|
||||
|
||||
await preflightDimMismatch(engine, !!opts.dryRun, embeddingColumn);
|
||||
|
||||
const result: EmbedResult = {
|
||||
embedded: 0,
|
||||
@@ -253,7 +264,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
for (const s of opts.slugs) {
|
||||
if (isAborted(opts.signal)) break; // #1737: stop the per-slug loop on abort
|
||||
try {
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal, embeddingColumn);
|
||||
} catch (e: unknown) {
|
||||
serr(` Error embedding ${s}: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
@@ -347,7 +358,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
catchUp: opts.catchUp,
|
||||
pacer,
|
||||
paceMaxConcurrency,
|
||||
}, opts.signal);
|
||||
}, opts.signal, embeddingColumn);
|
||||
} finally {
|
||||
// E1: surface pacing telemetry (human + structured) when pacing was on.
|
||||
const snap = pacer.snapshot();
|
||||
@@ -376,7 +387,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
return result;
|
||||
}
|
||||
if (opts.slug) {
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal, embeddingColumn);
|
||||
return result;
|
||||
}
|
||||
throw new Error('No embed target specified. Pass { slug }, { slugs }, { all }, or { stale }.');
|
||||
@@ -521,8 +532,13 @@ async function embedPage(
|
||||
result: EmbedResult,
|
||||
sourceId?: string,
|
||||
signal?: AbortSignal,
|
||||
embeddingColumn?: ResolvedColumn,
|
||||
) {
|
||||
const opts = sourceId ? { sourceId } : undefined;
|
||||
// #1262: write-side descriptor rides only on WRITE calls (upsertChunks).
|
||||
const chunkOpts = (sourceId || embeddingColumn)
|
||||
? { ...(sourceId && { sourceId }), ...(embeddingColumn && { embeddingColumn }) }
|
||||
: undefined;
|
||||
const page = await engine.getPage(slug, opts);
|
||||
if (!page) {
|
||||
throw new Error(`Page not found: ${slug}`);
|
||||
@@ -554,7 +570,7 @@ async function embedPage(
|
||||
}
|
||||
|
||||
if (inputs.length > 0) {
|
||||
await engine.upsertChunks(slug, inputs, opts);
|
||||
await engine.upsertChunks(slug, inputs, chunkOpts);
|
||||
chunks = await engine.getChunks(slug, opts);
|
||||
}
|
||||
}
|
||||
@@ -581,7 +597,7 @@ async function embedPage(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -589,7 +605,7 @@ async function embedPage(
|
||||
token_count: c.token_count || Math.ceil(c.chunk_text.length / 4),
|
||||
}));
|
||||
|
||||
await engine.upsertChunks(slug, updated, opts);
|
||||
await engine.upsertChunks(slug, updated, chunkOpts);
|
||||
// v0.41.31: stamp provenance so a later model/dims swap is detectable as
|
||||
// stale. embedPage is the per-slug path used by `gbrain embed <slug>` AND
|
||||
// by `gbrain sync`'s post-import embed step (runEmbedCore({slugs})).
|
||||
@@ -605,31 +621,6 @@ async function embedPage(
|
||||
slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry code-chunk metadata (language, symbol_name, symbol_type, line range,
|
||||
* parent scope, doc comment, qualified name) from a loaded Chunk back into a
|
||||
* ChunkInput destined for upsertChunks.
|
||||
*
|
||||
* Issue #769: every re-embed used to strip these fields, and upsertChunks
|
||||
* overwrites (does not COALESCE) the metadata columns from EXCLUDED, so
|
||||
* each pass clobbered code-def's primary index to NULL. Pulling the
|
||||
* preservation into one helper keeps the three re-embed call sites
|
||||
* (embedPage, embedAll non-stale, embedAllStale) in lock-step.
|
||||
*/
|
||||
function preserveCodeMetadata(loaded: any, base: ChunkInput): ChunkInput {
|
||||
return {
|
||||
...base,
|
||||
language: loaded.language ?? undefined,
|
||||
symbol_name: loaded.symbol_name ?? undefined,
|
||||
symbol_type: loaded.symbol_type ?? undefined,
|
||||
start_line: loaded.start_line ?? undefined,
|
||||
end_line: loaded.end_line ?? undefined,
|
||||
parent_symbol_path: loaded.parent_symbol_path ?? undefined,
|
||||
doc_comment: loaded.doc_comment ?? undefined,
|
||||
symbol_name_qualified: loaded.symbol_name_qualified ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function embedAll(
|
||||
engine: BrainEngine,
|
||||
staleOnly: boolean,
|
||||
@@ -647,6 +638,7 @@ async function embedAll(
|
||||
paceMaxConcurrency?: number;
|
||||
},
|
||||
signal?: AbortSignal,
|
||||
embeddingColumn?: ResolvedColumn,
|
||||
) {
|
||||
// v0.41.31: current embedding provenance signature. Stamped onto pages
|
||||
// when their chunks are (re)embedded so a later model/dimension swap is
|
||||
@@ -669,7 +661,7 @@ async function embedAll(
|
||||
// D7: thread sourceId so `gbrain embed --stale --source X` actually scopes.
|
||||
// v0.41.18.0 (A13): thread batchSize/priority/catchUp into the stale path.
|
||||
// #1737: thread the external abort signal so the cycle embed phase bails.
|
||||
return await embedAllStale(engine, sourceId, dryRun, result, onProgress, staleOpts, signature, signal);
|
||||
return await embedAllStale(engine, sourceId, dryRun, result, onProgress, staleOpts, signature, signal, embeddingColumn);
|
||||
}
|
||||
|
||||
// --all path: pacer (no-op when off). E-1: lower the worker count to the
|
||||
@@ -742,17 +734,18 @@ async function embedAll(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// Preserve ALL chunks, only update embeddings for stale ones.
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so re-embed
|
||||
// doesn't clobber language/symbol_name/symbol_type to NULL.
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
// Preserve ALL chunks, only update embeddings for stale ones
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: embeddingMap.get(c.chunk_index) ?? undefined,
|
||||
token_count: c.token_count || Math.ceil(c.chunk_text.length / 4),
|
||||
}));
|
||||
await observed(pacer, () => engine.upsertChunks(page.slug, updated, pageOpts));
|
||||
await observed(pacer, () => engine.upsertChunks(page.slug, updated, {
|
||||
...(pageSourceId && { sourceId: pageSourceId }),
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
}));
|
||||
// v0.41.31: stamp embedding provenance so a later model swap is
|
||||
// detectable as stale.
|
||||
await observed(pacer, () =>
|
||||
@@ -832,10 +825,16 @@ async function embedAllStale(
|
||||
},
|
||||
signature?: string,
|
||||
externalSignal?: AbortSignal,
|
||||
embeddingColumn?: ResolvedColumn,
|
||||
) {
|
||||
// D7: thread sourceId so source-scoped runs only count + visit
|
||||
// that source's NULL embeddings.
|
||||
const sourceOpt = sourceId ? { sourceId } : undefined;
|
||||
// #1262: the stale predicate follows the write-side column — without it an
|
||||
// alt-column brain would perpetually re-select (and re-pay for) chunks whose
|
||||
// target column is already populated.
|
||||
const sourceOpt = (sourceId || embeddingColumn)
|
||||
? { ...(sourceId && { sourceId }), ...(embeddingColumn && { embeddingColumn }) }
|
||||
: undefined;
|
||||
|
||||
// v0.41.31: re-embed pages whose embedding_signature drifted (model/dims
|
||||
// swap). dry-run must NOT mutate, so it counts signature-stale via the
|
||||
@@ -994,6 +993,7 @@ async function embedAllStale(
|
||||
afterUpdatedAt,
|
||||
}),
|
||||
...(sourceId && { sourceId }),
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
}),
|
||||
);
|
||||
if (batch.length === 0) {
|
||||
@@ -1039,17 +1039,17 @@ async function embedAllStale(
|
||||
for (let j = 0; j < stale.length; j++) {
|
||||
staleIdxToEmbedding.set(stale[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so the
|
||||
// autopilot --stale path doesn't clobber language/symbol_name/etc
|
||||
// to NULL on every cycle.
|
||||
const merged: ChunkInput[] = existing.map(c => preserveCodeMetadata(c, {
|
||||
const merged: ChunkInput[] = existing.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: staleIdxToEmbedding.get(c.chunk_index) ?? undefined,
|
||||
token_count: c.token_count || Math.ceil(c.chunk_text.length / 4),
|
||||
}));
|
||||
await observed(pacer, () => engine.upsertChunks(slug, merged, { sourceId: keySourceId }));
|
||||
await observed(pacer, () => engine.upsertChunks(slug, merged, {
|
||||
sourceId: keySourceId,
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
}));
|
||||
// v0.41.31: stamp provenance after the page's chunks are embedded —
|
||||
// but only when EVERY chunk was stale (fully re-embedded this pass).
|
||||
// A partially-stale page keeps preserved chunks of unknown/old
|
||||
@@ -1120,7 +1120,7 @@ async function embedAllStale(
|
||||
// as a clean run — re-running won't help until the underlying failure is fixed.
|
||||
if (staleOpts?.catchUp && !effectiveSignal.aborted && embedFailures > 0) {
|
||||
const remaining = await engine.countStaleChunks(
|
||||
signature ? { signature, ...(sourceId ? { sourceId } : {}) } : (sourceId ? { sourceId } : undefined),
|
||||
signature ? { signature, ...sourceOpt } : sourceOpt,
|
||||
);
|
||||
if (remaining > 0) {
|
||||
serr(`\n [embed] catch-up finished but ${remaining} chunk(s) remain stale after ${embedFailures} embed failure(s). These are not embeddable as-is; re-running won't clear them until the underlying error is resolved.`);
|
||||
|
||||
@@ -1651,7 +1651,7 @@ async function extractTimelineFromDB(
|
||||
* make re-extraction idempotent). EVERY processed page is stamped, including
|
||||
* zero-link pages — they WERE processed.
|
||||
*/
|
||||
export async function extractStaleFromDB(
|
||||
async function extractStaleFromDB(
|
||||
engine: BrainEngine,
|
||||
opts: {
|
||||
dryRun: boolean;
|
||||
|
||||
@@ -334,10 +334,6 @@ export async function scanIntegrity(
|
||||
if (!page) continue;
|
||||
// Skip grandfathered pages (opted out of brain-integrity enforcement)
|
||||
if ((page.frontmatter as Record<string, unknown> | undefined)?.validate === false) continue;
|
||||
// Skip code pages: indexed source files aren't prose. 'tweet' in an
|
||||
// identifier or comment is not a bare-tweet citation gap, and auto-repair
|
||||
// would inject wikilink brackets into source code.
|
||||
if (page.type === 'code') continue;
|
||||
pagesScanned++;
|
||||
bareHits.push(...findBareTweetHits(page.compiled_truth, slug));
|
||||
externalHits.push(...findExternalLinks(page.compiled_truth, slug));
|
||||
@@ -367,9 +363,6 @@ async function scanIntegrityBatch(
|
||||
// YAML) diverges from the sequential path's strict === false check. Intentional
|
||||
// — gbrain lint should reject stringly-typed validate at write time.
|
||||
const validateCondition = sql`AND (frontmatter->>'validate' IS NULL OR frontmatter->>'validate' != 'false')`;
|
||||
// Mirror of the sequential path's `page.type === 'code'` skip: code pages
|
||||
// (indexed source files) are never prose-integrity candidates.
|
||||
const codeCondition = sql`AND type IS DISTINCT FROM 'code'`;
|
||||
|
||||
// v0.32.8: scan ONE row per (source_id, slug) pair, not one per slug.
|
||||
// Pre-fix used DISTINCT ON (slug) which collapsed multi-source rows into
|
||||
@@ -379,7 +372,7 @@ async function scanIntegrityBatch(
|
||||
const rows = await sql`
|
||||
SELECT slug, compiled_truth, frontmatter
|
||||
FROM pages
|
||||
WHERE 1=1 ${typeCondition} ${validateCondition} ${codeCondition}
|
||||
WHERE 1=1 ${typeCondition} ${validateCondition}
|
||||
ORDER BY source_id, slug
|
||||
LIMIT ${limit}
|
||||
`;
|
||||
@@ -468,9 +461,6 @@ async function cmdAuto(args: string[]): Promise<void> {
|
||||
|
||||
const page = await engine.getPage(slug, { sourceId: source_id });
|
||||
if (!page) continue;
|
||||
// Never auto-repair code pages — injecting tweet citations into
|
||||
// indexed source files corrupts them. Same gate as scanIntegrity.
|
||||
if (page.type === 'code') continue;
|
||||
|
||||
pagesProcessed++;
|
||||
progress.tick(1, slug);
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
/**
|
||||
* gbrain maintain — conservative self-healing maintenance.
|
||||
*
|
||||
* This command automates the safe parts of the operator runbook:
|
||||
* - stale link/timeline extraction
|
||||
* - stale per-source dream cycles when doctor reports cycle_freshness
|
||||
*
|
||||
* It deliberately does NOT mutate source files, apply schema-pack upgrades, or
|
||||
* invent semantic hub links. Those need review or a separate command with an
|
||||
* auditable proposal surface.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { BrainHealth } from '../core/types.ts';
|
||||
import { buildChecks, computeDoctorReport, type DoctorReport, type Check } from './doctor.ts';
|
||||
import { extractStaleFromDB } from './extract.ts';
|
||||
import { runCycle, type CycleReport } from '../core/cycle.ts';
|
||||
|
||||
type ActionStatus = 'ok' | 'would_apply' | 'applied' | 'blocked' | 'skipped';
|
||||
|
||||
export interface MaintenanceAction {
|
||||
name: string;
|
||||
status: ActionStatus;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface MaintainOptions {
|
||||
json: boolean;
|
||||
safe: boolean;
|
||||
dryRun: boolean;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
export interface MaintainReport {
|
||||
mode: 'dry-run' | 'safe';
|
||||
before: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
actions: MaintenanceAction[];
|
||||
after: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
}
|
||||
|
||||
export function parseMaintainArgs(args: string[]): MaintainOptions {
|
||||
const safe = args.includes('--safe');
|
||||
return {
|
||||
json: args.includes('--json'),
|
||||
safe,
|
||||
dryRun: args.includes('--dry-run') || !safe,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractCycleFreshnessSourceIds(checks: Check[]): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const check of checks) {
|
||||
if (check.name !== 'cycle_freshness' || check.status === 'ok') continue;
|
||||
const re = /Source '([^']+)' last cycled/g;
|
||||
for (const match of check.message.matchAll(re)) {
|
||||
const id = match[1]?.trim();
|
||||
if (id) ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids].sort();
|
||||
}
|
||||
|
||||
async function buildDoctorReport(engine: BrainEngine): Promise<DoctorReport> {
|
||||
const checks = await buildChecks(engine, ['--json', '--scope=brain']);
|
||||
return computeDoctorReport(checks);
|
||||
}
|
||||
|
||||
async function runStaleExtraction(
|
||||
engine: BrainEngine,
|
||||
beforeHealth: BrainHealth,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction> {
|
||||
if (beforeHealth.stale_pages <= 0) {
|
||||
return { name: 'extract_stale', status: 'ok', message: 'No stale pages.' };
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'would_apply',
|
||||
message: `Would run DB-backed stale extraction for ${beforeHealth.stale_pages} page(s).`,
|
||||
details: { stale_pages: beforeHealth.stale_pages },
|
||||
};
|
||||
}
|
||||
|
||||
const result = await extractStaleFromDB(engine, {
|
||||
dryRun: false,
|
||||
jsonMode: false,
|
||||
includeFrontmatter: false,
|
||||
catchUp: false,
|
||||
});
|
||||
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'applied',
|
||||
message: `Processed ${result.pagesProcessed} stale page(s); ${result.staleRemaining} remain.`,
|
||||
details: {
|
||||
links_created: result.linksCreated,
|
||||
timeline_created: result.timelineCreated,
|
||||
pages_processed: result.pagesProcessed,
|
||||
stale_remaining: result.staleRemaining,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runCycleFreshnessMaintenance(
|
||||
engine: BrainEngine,
|
||||
beforeDoctor: DoctorReport,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction[]> {
|
||||
const sourceIds = extractCycleFreshnessSourceIds(beforeDoctor.checks);
|
||||
if (sourceIds.length === 0) {
|
||||
return [{ name: 'cycle_freshness', status: 'ok', message: 'All sources cycled recently.' }];
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return sourceIds.map((sourceId) => ({
|
||||
name: 'cycle_freshness',
|
||||
status: 'would_apply',
|
||||
message: `Would run source-scoped dream cycle for ${sourceId}.`,
|
||||
details: { source_id: sourceId },
|
||||
}));
|
||||
}
|
||||
|
||||
const sources = await engine.listAllSources();
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
for (const sourceId of sourceIds) {
|
||||
const source = sources.find((s) => s.id === sourceId);
|
||||
const localPath = source?.local_path ?? null;
|
||||
const brainDir = localPath && existsSync(localPath) ? localPath : null;
|
||||
const report: CycleReport = await runCycle(engine, {
|
||||
brainDir,
|
||||
dryRun: false,
|
||||
pull: false,
|
||||
sourceId,
|
||||
});
|
||||
actions.push({
|
||||
name: 'cycle_freshness',
|
||||
status: report.status === 'failed' ? 'blocked' : 'applied',
|
||||
message: `Ran source-scoped dream cycle for ${sourceId}: ${report.status}.`,
|
||||
details: {
|
||||
source_id: sourceId,
|
||||
brain_dir: brainDir,
|
||||
cycle_status: report.status,
|
||||
phases: report.phases.map((p) => ({ phase: p.phase, status: p.status })),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
export async function runMaintain(engine: BrainEngine, args: string[]): Promise<MaintainReport | void> {
|
||||
const opts = parseMaintainArgs(args);
|
||||
if (opts.help) {
|
||||
console.log(`Usage: gbrain maintain [--safe] [--dry-run] [--json]
|
||||
|
||||
Conservative self-healing maintenance.
|
||||
|
||||
Modes:
|
||||
--dry-run Preview safe actions without writes. Default when --safe is absent.
|
||||
--safe Apply safe actions: stale extraction and source cycle freshness.
|
||||
--json Emit a structured before/action/after report.
|
||||
|
||||
Not auto-applied:
|
||||
source-file frontmatter fixes, schema-pack upgrades, atom-pack changes,
|
||||
semantic hub-link guesses, and destructive cleanup.
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
const beforeHealth = await engine.getHealth();
|
||||
const beforeDoctor = await buildDoctorReport(engine);
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
actions.push(await runStaleExtraction(engine, beforeHealth, opts.dryRun));
|
||||
actions.push(...await runCycleFreshnessMaintenance(engine, beforeDoctor, opts.dryRun));
|
||||
|
||||
const afterHealth = await engine.getHealth();
|
||||
const afterDoctor = await buildDoctorReport(engine);
|
||||
const report: MaintainReport = {
|
||||
mode: opts.dryRun ? 'dry-run' : 'safe',
|
||||
before: { health: beforeHealth, doctor: beforeDoctor },
|
||||
actions,
|
||||
after: { health: afterHealth, doctor: afterDoctor },
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
printMaintainReport(report);
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
function printMaintainReport(report: MaintainReport): void {
|
||||
console.log(`GBrain maintain (${report.mode})`);
|
||||
console.log(
|
||||
`Before: brain_score=${Math.round(report.before.health.brain_score)}/100 ` +
|
||||
`stale=${report.before.health.stale_pages} islands=${report.before.health.orphan_pages} ` +
|
||||
`doctor=${report.before.doctor.status}`,
|
||||
);
|
||||
for (const action of report.actions) {
|
||||
console.log(` ${action.status}: ${action.name} — ${action.message}`);
|
||||
}
|
||||
console.log(
|
||||
`After: brain_score=${Math.round(report.after.health.brain_score)}/100 ` +
|
||||
`stale=${report.after.health.stale_pages} islands=${report.after.health.orphan_pages} ` +
|
||||
`doctor=${report.after.doctor.status}`,
|
||||
);
|
||||
if (report.mode === 'dry-run') {
|
||||
console.log('Run `gbrain maintain --safe` to apply safe actions.');
|
||||
}
|
||||
}
|
||||
+55
-10
@@ -15,11 +15,6 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { createProgress, startHeartbeat } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import {
|
||||
shouldExcludeFromOrphanReporting,
|
||||
loadOrphanPolicyOverrides,
|
||||
type OrphanPolicyOverrides,
|
||||
} from '../core/orphan-policy.ts';
|
||||
|
||||
// --- Types ---
|
||||
|
||||
@@ -37,14 +32,65 @@ export interface OrphanResult {
|
||||
excluded: number;
|
||||
}
|
||||
|
||||
// --- Filter constants ---
|
||||
|
||||
/** Slug suffixes that are always auto-generated root files */
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
/** Page slugs that are pseudo-pages by convention */
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
/** Slug segment that marks raw sources */
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
/** Slug prefixes where no inbound links is expected */
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'openclaw/config/',
|
||||
];
|
||||
|
||||
/** First slug segments where no inbound links is expected */
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set([
|
||||
'scratch',
|
||||
'thoughts',
|
||||
'catalog',
|
||||
'entities',
|
||||
'raw',
|
||||
'atoms',
|
||||
'skills',
|
||||
]);
|
||||
|
||||
// --- Filter logic ---
|
||||
|
||||
/**
|
||||
* Returns true if a slug should be excluded from orphan reporting by default.
|
||||
* These are pages where having no inbound links is expected / not a content problem.
|
||||
*/
|
||||
export function shouldExclude(slug: string, overrides?: OrphanPolicyOverrides): boolean {
|
||||
return shouldExcludeFromOrphanReporting(slug, overrides);
|
||||
export function shouldExclude(slug: string): boolean {
|
||||
// Pseudo-pages (exact match)
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
// Auto-generated suffix patterns
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
// Raw source slugs
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
|
||||
// Deny-prefix slugs
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
// First-segment exclusions
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,7 +156,6 @@ export async function findOrphans(
|
||||
let allOrphans: { slug: string; title: string; domain: string | null }[];
|
||||
let total: number;
|
||||
let excludedAll: number;
|
||||
const overrides = includePseudo ? undefined : await loadOrphanPolicyOverrides(engine);
|
||||
try {
|
||||
allOrphans = await engine.findOrphanPages(
|
||||
sourceIds ? { sourceIds } : sourceId ? { sourceId } : undefined,
|
||||
@@ -139,7 +184,7 @@ export async function findOrphans(
|
||||
total = liveRows.length;
|
||||
excludedAll = includePseudo
|
||||
? 0
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug, overrides) ? 1 : 0), 0);
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug) ? 1 : 0), 0);
|
||||
} finally {
|
||||
stopHb();
|
||||
progress.finish();
|
||||
@@ -147,7 +192,7 @@ export async function findOrphans(
|
||||
|
||||
const filtered = includePseudo
|
||||
? allOrphans
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug, overrides));
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug));
|
||||
|
||||
const orphans: OrphanPage[] = filtered.map(row => ({
|
||||
slug: row.slug,
|
||||
|
||||
@@ -576,9 +576,16 @@ async function runInlineCostGate(
|
||||
|
||||
// Stale backlog: cheap single SQL; fail-open to 0 so a transient DB hiccup
|
||||
// never blocks the sync. Signature-aware (model/dims swap surfaces here).
|
||||
// #1262: follow the write-side embedding column — otherwise an alt-column
|
||||
// brain's fully-embedded corpus counts as phantom backlog on every gate.
|
||||
let staleChars = 0;
|
||||
try {
|
||||
staleChars = await engine.sumStaleChunkChars({ signature: currentEmbeddingSignature() });
|
||||
const { resolveWriteColumnForEngine } = await import('../core/search/embedding-column.ts');
|
||||
const embeddingColumn = await resolveWriteColumnForEngine(engine);
|
||||
staleChars = await engine.sumStaleChunkChars({
|
||||
signature: currentEmbeddingSignature(),
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
});
|
||||
} catch {
|
||||
staleChars = 0;
|
||||
}
|
||||
|
||||
@@ -54,7 +54,6 @@ import {
|
||||
import {
|
||||
generatePerChunkSynopsis,
|
||||
SYNOPSIS_PROMPT_VERSION,
|
||||
SYNOPSIS_DOC_MAX_CHARS,
|
||||
type GeneratePerChunkSynopsisResult,
|
||||
} from './page-summary.ts';
|
||||
import {
|
||||
@@ -62,6 +61,7 @@ import {
|
||||
type SynopsisFailureKind,
|
||||
} from './audit-synopsis.ts';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { resolveWriteColumnForEngine } from './search/embedding-column.ts';
|
||||
import type { ChunkInput, CRMode, Page } from './types.ts';
|
||||
import type { SourceRow } from './sources-ops.ts';
|
||||
|
||||
@@ -104,17 +104,8 @@ function getEmbeddingModelTag(): string {
|
||||
export function computeCorpusGeneration(args: {
|
||||
crMode: CRMode;
|
||||
haikuModel: string;
|
||||
/**
|
||||
* Resolved `SYNOPSIS_DOC_MAX_CHARS` for per_chunk_synopsis runs. When
|
||||
* present, folded into the hash so changes to
|
||||
* `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` invalidate the prior cache cleanly.
|
||||
* Omit for `crMode !== 'per_chunk_synopsis'` — title / none modes
|
||||
* don't consult the cap and the field stays out of the hash for
|
||||
* back-compat with pre-cap embeddings.
|
||||
*/
|
||||
synopsisDocMaxChars?: number;
|
||||
}): string {
|
||||
const h = createHash('sha256')
|
||||
return createHash('sha256')
|
||||
.update(args.crMode)
|
||||
.update('|')
|
||||
.update(String(SYNOPSIS_PROMPT_VERSION))
|
||||
@@ -123,11 +114,9 @@ export function computeCorpusGeneration(args: {
|
||||
.update('|')
|
||||
.update(String(TITLE_WRAPPER_VERSION))
|
||||
.update('|')
|
||||
.update(getEmbeddingModelTag());
|
||||
if (args.synopsisDocMaxChars !== undefined) {
|
||||
h.update('|doc_cap=').update(String(args.synopsisDocMaxChars));
|
||||
}
|
||||
return h.digest('hex').slice(0, 16);
|
||||
.update(getEmbeddingModelTag())
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -265,11 +254,7 @@ export async function reembedPageWithContextualRetrieval(
|
||||
args.pageSlug,
|
||||
args.sourceId,
|
||||
resolution.mode,
|
||||
computeCorpusGeneration({
|
||||
crMode: resolution.mode,
|
||||
haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL,
|
||||
synopsisDocMaxChars: resolution.mode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
|
||||
}),
|
||||
computeCorpusGeneration({ crMode: resolution.mode, haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL }),
|
||||
);
|
||||
return { kind: 'skipped', reason: 'no_chunks' };
|
||||
}
|
||||
@@ -298,14 +283,17 @@ export async function reembedPageWithContextualRetrieval(
|
||||
const corpus_generation = computeCorpusGeneration({
|
||||
crMode: attemptMode,
|
||||
haikuModel,
|
||||
synopsisDocMaxChars: attemptMode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
|
||||
});
|
||||
|
||||
// ── PHASE 2: single DB transaction ───────────────────────────
|
||||
try {
|
||||
// #1262: contextual re-embeds write TEXT embeddings — thread the
|
||||
// caller-resolved write column like every other embed path.
|
||||
const embeddingColumn = await resolveWriteColumnForEngine(args.engine);
|
||||
await args.engine.transaction(async (tx) => {
|
||||
await tx.upsertChunks(args.pageSlug, phase1.embeddedChunks, {
|
||||
sourceId: args.sourceId,
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
});
|
||||
await tx.updatePageContextualRetrievalState(
|
||||
args.pageSlug,
|
||||
|
||||
+13
-2
@@ -18,7 +18,7 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import type { ChunkInput, ResolvedColumn } from './types.ts';
|
||||
import { embedBatchWithBackoff } from '../commands/embed.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
@@ -61,6 +61,13 @@ export interface EmbedStaleOpts {
|
||||
* Omit to keep the legacy `embedding IS NULL`-only behavior.
|
||||
*/
|
||||
embeddingSignature?: string;
|
||||
/**
|
||||
* #1262: caller-resolved write-side embedding column. Threaded into BOTH
|
||||
* listStaleChunks (staleness predicate) and upsertChunks (write target) so
|
||||
* an alt-column brain converges instead of re-selecting embedded rows.
|
||||
* Resolve at the boundary via `resolveWriteColumnForEngine()`.
|
||||
*/
|
||||
embeddingColumn?: ResolvedColumn;
|
||||
/**
|
||||
* DB-contention pacer (paced-backfill). When enabled it (a) supplies the
|
||||
* worker count via the caller passing `concurrency = bundle.maxConcurrency`
|
||||
@@ -156,6 +163,7 @@ export async function embedStaleForSource(
|
||||
afterPageId,
|
||||
afterChunkIndex,
|
||||
sourceId,
|
||||
...(opts.embeddingColumn && { embeddingColumn: opts.embeddingColumn }),
|
||||
}),
|
||||
);
|
||||
if (batch.length === 0) {
|
||||
@@ -223,7 +231,10 @@ export async function embedStaleForSource(
|
||||
doc_comment: c.doc_comment ?? undefined,
|
||||
symbol_name_qualified: c.symbol_name_qualified ?? undefined,
|
||||
}));
|
||||
await observed(pacer, () => engine.upsertChunks(slug, merged, { sourceId: keySourceId }));
|
||||
await observed(pacer, () => engine.upsertChunks(slug, merged, {
|
||||
sourceId: keySourceId,
|
||||
...(opts.embeddingColumn && { embeddingColumn: opts.embeddingColumn }),
|
||||
}));
|
||||
// v0.41.31: stamp provenance only when EVERY chunk was stale (fully
|
||||
// re-embedded this pass) — a partially-stale page keeps preserved
|
||||
// chunks of unknown provenance, so don't claim current. After the
|
||||
|
||||
+22
-3
@@ -12,6 +12,7 @@ import type {
|
||||
BrainStats, BrainHealth,
|
||||
IngestLogEntry, IngestLogInput,
|
||||
EngineConfig,
|
||||
ResolvedColumn,
|
||||
CodeEdgeInput, CodeEdgeResult,
|
||||
EvalCandidate, EvalCandidateInput,
|
||||
EvalCaptureFailure, EvalCaptureFailureReason,
|
||||
@@ -987,8 +988,13 @@ export interface BrainEngine {
|
||||
* — Postgres rolls back automatically on conn drop, so commit-ambiguous
|
||||
* failure replays to the same end state. Callers MUST NOT wrap externally;
|
||||
* see {@link BatchOpts} retry-contract block.
|
||||
*
|
||||
* `opts.embeddingColumn` (optional) selects the content_chunks column that
|
||||
* receives TEXT embeddings (#1262). The caller resolves the descriptor at
|
||||
* the import/embed boundary via `resolveWriteColumn()`; engines never read
|
||||
* config or choose columns themselves. Omitted => legacy `embedding`.
|
||||
*/
|
||||
upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string } & BatchOpts): Promise<void>;
|
||||
upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string; embeddingColumn?: ResolvedColumn } & BatchOpts): Promise<void>;
|
||||
/**
|
||||
* Read every chunk for a page. `opts.sourceId` source-scopes the page
|
||||
* lookup; without it, multi-source brains return chunks from every
|
||||
@@ -1005,8 +1011,13 @@ export interface BrainEngine {
|
||||
* counts across every source in the brain. Operators running
|
||||
* `gbrain embed --stale --source media-corpus` expect only that
|
||||
* source's NULLs touched; the caller threads `sourceId` here.
|
||||
*
|
||||
* `opts.embeddingColumn` switches the staleness predicate from the legacy
|
||||
* `embedding` column to the resolved write-side column, so alt-column
|
||||
* brains do not perpetually re-select rows whose target column is already
|
||||
* populated (#1262). Must match the eventual upsertChunks target.
|
||||
*/
|
||||
countStaleChunks(opts?: { sourceId?: string; signature?: string }): Promise<number>;
|
||||
countStaleChunks(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number>;
|
||||
/**
|
||||
* Sum of LENGTH(chunk_text) over stale chunks — the character-count
|
||||
* backlog the embed phase / embed-backfill will process. Sibling of
|
||||
@@ -1020,8 +1031,13 @@ export interface BrainEngine {
|
||||
* model signature (a model/dims swap). NULL signature is GRANDFATHERED
|
||||
* (never counted) so the post-migration corpus isn't flagged en masse.
|
||||
* Omit `signature` for the legacy `embedding IS NULL`-only count.
|
||||
*
|
||||
* `opts.embeddingColumn` switches the staleness predicate to the resolved
|
||||
* write-side column (#1262) — same contract as countStaleChunks — so the
|
||||
* sync cost gate doesn't count an alt-column brain's fully-embedded corpus
|
||||
* as phantom backlog.
|
||||
*/
|
||||
sumStaleChunkChars(opts?: { sourceId?: string; signature?: string }): Promise<number>;
|
||||
sumStaleChunkChars(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number>;
|
||||
/**
|
||||
* Stamp `pages.embedding_signature = signature` for one page. Called after
|
||||
* a page's chunks are (re)embedded so a later model swap can detect it as
|
||||
@@ -1069,6 +1085,9 @@ export interface BrainEngine {
|
||||
// both round-trip TIMESTAMPTZ as Date | string; ISO string is the
|
||||
// common denominator on the wire).
|
||||
afterUpdatedAt?: string | null;
|
||||
// #1262: staleness predicate targets this column when set (must match
|
||||
// countStaleChunks and the eventual upsertChunks write target).
|
||||
embeddingColumn?: ResolvedColumn;
|
||||
}): Promise<StaleChunkRow[]>;
|
||||
/**
|
||||
* Delete every chunk for a page. Internal page-id lookup is sourceId-scoped
|
||||
|
||||
+25
-9
@@ -10,7 +10,8 @@ import { findChunkForOffset } from './chunkers/edge-extractor.ts';
|
||||
import { extractCodeRefs, imageOfCandidates } from './link-extraction.ts';
|
||||
import { embedBatch, embedMultimodal, currentEmbeddingSignature } from './embedding.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
|
||||
import type { ChunkInput, PageInput, PageType } from './types.ts';
|
||||
import type { ChunkInput, PageInput, PageType, ResolvedColumn } from './types.ts';
|
||||
import { resolveWriteColumnForEngine } from './search/embedding-column.ts';
|
||||
import { computeEffectiveDate } from './effective-date.ts';
|
||||
import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts';
|
||||
import { logSlugFallback } from './audit-slug-fallback.ts';
|
||||
@@ -733,11 +734,6 @@ export async function importFromContent(
|
||||
: computeCorpusGeneration({
|
||||
crMode: effectiveCRMode,
|
||||
haikuModel: 'anthropic:claude-haiku-4-5-20251001',
|
||||
// Inline import-file path never uses per_chunk_synopsis (refuses
|
||||
// upstream); pass undefined so the doc-cap field stays out of
|
||||
// the hash here. Per_chunk_synopsis runs through the Minion
|
||||
// backfill handler which threads SYNOPSIS_DOC_MAX_CHARS through
|
||||
// the service layer.
|
||||
});
|
||||
|
||||
// Transaction wraps all DB writes. Every per-page tx call carries the
|
||||
@@ -745,6 +741,14 @@ export async function importFromContent(
|
||||
// schema DEFAULT — required for multi-source brains; harmless ('default')
|
||||
// for single-source callers.
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
// #1262: resolve the write-side embedding column once (merged config +
|
||||
// gateway model) BEFORE the transaction; the descriptor rides only on
|
||||
// upsertChunks so text embeddings land in the registered column.
|
||||
const chunkWriteColumn = await resolveWriteColumnForEngine(engine);
|
||||
const chunkOpts: { sourceId?: string; embeddingColumn?: ResolvedColumn } | undefined =
|
||||
(sourceId || chunkWriteColumn)
|
||||
? { ...(sourceId && { sourceId }), ...(chunkWriteColumn && { embeddingColumn: chunkWriteColumn }) }
|
||||
: undefined;
|
||||
await engine.transaction(async (tx) => {
|
||||
if (existing) await tx.createVersion(slug, txOpts);
|
||||
|
||||
@@ -829,7 +833,7 @@ export async function importFromContent(
|
||||
}
|
||||
|
||||
if (chunks.length > 0) {
|
||||
await tx.upsertChunks(slug, chunks, txOpts);
|
||||
await tx.upsertChunks(slug, chunks, chunkOpts);
|
||||
// v0.41.31: stamp embedding provenance when this import actually
|
||||
// embedded (not --no-embed), so a later model/dims swap is detectable
|
||||
// as stale via embed --stale. The deferred/backfill + per-slug embed
|
||||
@@ -1069,6 +1073,12 @@ export async function importCodeFile(
|
||||
const title = `${relativePath} (${lang})`;
|
||||
const sourceId = opts.sourceId;
|
||||
const txOpts = sourceId ? { sourceId } : undefined;
|
||||
// #1262: write-side embedding column descriptor (rides only on upsertChunks).
|
||||
const chunkWriteColumn = await resolveWriteColumnForEngine(engine);
|
||||
const chunkOpts: { sourceId?: string; embeddingColumn?: ResolvedColumn } | undefined =
|
||||
(sourceId || chunkWriteColumn)
|
||||
? { ...(sourceId && { sourceId }), ...(chunkWriteColumn && { embeddingColumn: chunkWriteColumn }) }
|
||||
: undefined;
|
||||
|
||||
const byteLength = Buffer.byteLength(content, 'utf-8');
|
||||
if (byteLength > MAX_FILE_SIZE) {
|
||||
@@ -1188,7 +1198,7 @@ export async function importCodeFile(
|
||||
await tx.addTag(slug, lang, txOpts);
|
||||
|
||||
if (chunks.length > 0) {
|
||||
await tx.upsertChunks(slug, chunks, txOpts);
|
||||
await tx.upsertChunks(slug, chunks, chunkOpts);
|
||||
// v0.41.31: stamp embedding provenance ONLY when every chunk was
|
||||
// freshly embedded with the current model this call (no reuse-by-hash
|
||||
// carrying old-model vectors). Mixed pages stay unstamped rather than
|
||||
@@ -1337,6 +1347,12 @@ export async function withImportTransaction(
|
||||
): Promise<void> {
|
||||
const sourceId = spec.sourceId ?? 'default';
|
||||
const txOpts = spec.sourceId ? { sourceId: spec.sourceId } : undefined;
|
||||
// #1262: write-side embedding column descriptor (rides only on upsertChunks).
|
||||
const chunkWriteColumn = await resolveWriteColumnForEngine(engine);
|
||||
const chunkOpts: { sourceId?: string; embeddingColumn?: ResolvedColumn } | undefined =
|
||||
(spec.sourceId || chunkWriteColumn)
|
||||
? { ...(spec.sourceId && { sourceId: spec.sourceId }), ...(chunkWriteColumn && { embeddingColumn: chunkWriteColumn }) }
|
||||
: undefined;
|
||||
await engine.transaction(async (tx) => {
|
||||
if (spec.hadExisting) await tx.createVersion(spec.slug, txOpts);
|
||||
await tx.putPage(spec.slug, spec.page, txOpts);
|
||||
@@ -1352,7 +1368,7 @@ export async function withImportTransaction(
|
||||
}
|
||||
if (spec.chunks !== undefined) {
|
||||
if (spec.chunks.length > 0) {
|
||||
await tx.upsertChunks(spec.slug, spec.chunks, txOpts);
|
||||
await tx.upsertChunks(spec.slug, spec.chunks, chunkOpts);
|
||||
} else {
|
||||
await tx.deleteChunks(spec.slug, txOpts);
|
||||
}
|
||||
|
||||
@@ -489,22 +489,7 @@ export async function extractPageLinks(
|
||||
// text inside `[[...]]` before any `|`), NOT the display alias
|
||||
// (ref.name = match[2]). `[[struktura|the project]]` must resolve
|
||||
// `struktura`, not "the project". The display text is for context only.
|
||||
//
|
||||
// The literal may be path-qualified (`[[notes/struktura]]`). The FS
|
||||
// path (resolveSlugAll) strips the dirname before its basename lookup,
|
||||
// but this path passed the raw literal to an index keyed by final
|
||||
// segments only — so every slash-containing wikilink outside
|
||||
// DIR_PATTERN silently resolved to nothing. Query by the final
|
||||
// segment, then use the written path as a disambiguation filter
|
||||
// (the analogue of the FS ancestor walk honoring the written path):
|
||||
// a match must end with the literal, so `[[notes/struktura]]` can
|
||||
// resolve to `vault/notes/struktura` but never to `wiki/struktura`.
|
||||
const slashIdx = ref.slug.lastIndexOf('/');
|
||||
const basename = slashIdx === -1 ? ref.slug : ref.slug.slice(slashIdx + 1);
|
||||
let matches = await resolver.resolveBasenameMatches(basename);
|
||||
if (slashIdx !== -1) {
|
||||
matches = matches.filter(m => m === ref.slug || m.endsWith(`/${ref.slug}`));
|
||||
}
|
||||
const matches = await resolver.resolveBasenameMatches(ref.slug);
|
||||
if (matches.length === 0) continue;
|
||||
const idx = content.indexOf(ref.slug);
|
||||
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
|
||||
|
||||
@@ -35,6 +35,7 @@ import { tryAcquireDbLock } from '../../db-lock.ts';
|
||||
import { BudgetTracker, BudgetExhausted } from '../../budget/budget-tracker.ts';
|
||||
import { withBudgetTracker } from '../../ai/gateway.ts';
|
||||
import { embedStaleForSource } from '../../embed-stale.ts';
|
||||
import { resolveWriteColumnForEngine } from '../../search/embedding-column.ts';
|
||||
import { currentEmbeddingSignature } from '../../embedding.ts';
|
||||
import { type DbPacer, createDbPacer, createNoopPacer } from '../../db-pacer.ts';
|
||||
import { resolvePaceMode, loadPaceModeConfig, readPaceEnv } from '../../pace-mode.ts';
|
||||
@@ -164,12 +165,16 @@ export function makeEmbedBackfillHandler(engine: BrainEngine) {
|
||||
// the supervisor, so pacing it is the headline win.
|
||||
const { pacer, concurrency } = await resolveBackfillPacer(engine, job.data);
|
||||
|
||||
// #1262: resolve the write-side embedding column once at the job boundary.
|
||||
const embeddingColumn = await resolveWriteColumnForEngine(engine);
|
||||
|
||||
try {
|
||||
const result = await withBudgetTracker(tracker, async () =>
|
||||
embedStaleForSource(engine, sourceId, {
|
||||
batchSize,
|
||||
signal: job.signal,
|
||||
pacer,
|
||||
...(embeddingColumn && { embeddingColumn }),
|
||||
...(concurrency !== undefined && { concurrency }),
|
||||
// v0.41.31: re-embed pages whose model signature drifted + stamp
|
||||
// provenance as chunks land.
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
/**
|
||||
* Shared orphan-reporting exclusion policy.
|
||||
*
|
||||
* These are pages where "no inbound links" is expected and should not count
|
||||
* against health. Keep this in core so the CLI orphan report and engine health
|
||||
* dashboard cannot drift.
|
||||
*
|
||||
* Defaults are GBrain-wide conventions only. Brain-specific exclusions
|
||||
* (private folder names, one-off fixture slugs) belong in the brain's own
|
||||
* config, not here:
|
||||
*
|
||||
* gbrain config set orphans.exclude_prefixes "my-private-folder/,archive/"
|
||||
* gbrain config set orphans.exclude_slugs "some-one-off-page"
|
||||
*/
|
||||
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'_templates/',
|
||||
'openclaw/config/',
|
||||
'extracts/',
|
||||
];
|
||||
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set([
|
||||
'scratch',
|
||||
'thoughts',
|
||||
'catalog',
|
||||
'entities',
|
||||
'raw',
|
||||
'atoms',
|
||||
'skills',
|
||||
'dreaming',
|
||||
'daily',
|
||||
]);
|
||||
|
||||
const ROOT_DATE_SLUG = /^\d{4}-\d{2}-\d{2}(?:-.+)?$/;
|
||||
|
||||
function isAgentWorkspaceConvention(slug: string): boolean {
|
||||
if (!slug.startsWith('agents/')) return false;
|
||||
if (slug.includes('/memory/dreaming/')) return true;
|
||||
return /^agents\/[^/]+\/(?:agents|identity|soul|tools|user|heartbeat|dreams|dormant)$/.test(slug);
|
||||
}
|
||||
|
||||
/** Per-brain additions to the convention defaults (from config). */
|
||||
export interface OrphanPolicyOverrides {
|
||||
excludePrefixes?: string[];
|
||||
excludeSlugs?: string[];
|
||||
}
|
||||
|
||||
/** Config keys for per-brain orphan exclusions (comma-separated values). */
|
||||
export const ORPHAN_EXCLUDE_PREFIXES_KEY = 'orphans.exclude_prefixes';
|
||||
export const ORPHAN_EXCLUDE_SLUGS_KEY = 'orphans.exclude_slugs';
|
||||
|
||||
function parseList(value: string | null): string[] {
|
||||
if (!value) return [];
|
||||
return value.split(',').map(s => s.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load per-brain orphan exclusions from the brain config table. Callers with
|
||||
* an engine in hand (getHealth, `gbrain orphans`) pass the result as the
|
||||
* second argument to shouldExcludeFromOrphanReporting.
|
||||
*/
|
||||
export async function loadOrphanPolicyOverrides(
|
||||
engine: { getConfig(key: string): Promise<string | null> },
|
||||
): Promise<OrphanPolicyOverrides> {
|
||||
const [prefixes, slugs] = await Promise.all([
|
||||
engine.getConfig(ORPHAN_EXCLUDE_PREFIXES_KEY),
|
||||
engine.getConfig(ORPHAN_EXCLUDE_SLUGS_KEY),
|
||||
]);
|
||||
return { excludePrefixes: parseList(prefixes), excludeSlugs: parseList(slugs) };
|
||||
}
|
||||
|
||||
export function shouldExcludeFromOrphanReporting(
|
||||
slug: string,
|
||||
overrides?: OrphanPolicyOverrides,
|
||||
): boolean {
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
if (slug.includes('/daily/')) return true;
|
||||
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
if (ROOT_DATE_SLUG.test(slug)) return true;
|
||||
|
||||
if (slug.startsWith('_brain-')) return true;
|
||||
|
||||
if (isAgentWorkspaceConvention(slug)) return true;
|
||||
|
||||
if (overrides) {
|
||||
if (overrides.excludeSlugs?.includes(slug)) return true;
|
||||
for (const prefix of overrides.excludePrefixes ?? []) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -44,33 +44,6 @@ const HAIKU_MAX_TOKENS = 200;
|
||||
/** Default model when caller doesn't override. Resolves through the gateway. */
|
||||
const DEFAULT_SYNOPSIS_MODEL = 'anthropic:claude-haiku-4-5-20251001';
|
||||
|
||||
/**
|
||||
* Hard cap on `documentText` length (chars) before send.
|
||||
*
|
||||
* 2026-05-25 fix wave: small local chat models (Gemma 4 E2B, Qwen3 4B) get
|
||||
* dramatically slower on long contexts even with 131K-token windows declared.
|
||||
* A 73K-char page synopsis on Gemma 4 E2B takes 60-120s, exceeding the
|
||||
* worker's default 30s `lockDuration` and tripping `lock-lost` errors.
|
||||
*
|
||||
* Truncate to a budget that fits a small model's effective throughput while
|
||||
* preserving enough document context for the synopsis to be useful. Truncates
|
||||
* the TAIL because the head (title, frontmatter, intro) carries the
|
||||
* document-level anchor the synopsis needs.
|
||||
*
|
||||
* Override per workload via `GBRAIN_SYNOPSIS_DOC_MAX_CHARS`. Default 32768
|
||||
* (~8K tokens at 4 chars/tok) keeps small-model synopsis under ~30s.
|
||||
* Anthropic Haiku is unaffected at this cap; bump higher when running
|
||||
* frontier models if you want richer document anchoring.
|
||||
*/
|
||||
export const SYNOPSIS_DOC_MAX_CHARS = (() => {
|
||||
const env = process.env.GBRAIN_SYNOPSIS_DOC_MAX_CHARS;
|
||||
if (env && /^\d+$/.test(env)) {
|
||||
const n = parseInt(env, 10);
|
||||
if (n >= 512 && n <= 1_048_576) return n;
|
||||
}
|
||||
return 32768;
|
||||
})();
|
||||
|
||||
/**
|
||||
* Synopsis prompt version. Folded into corpus_generation so prompt edits
|
||||
* invalidate prior embeddings via the v0.40.3.0 query_cache.page_generations
|
||||
@@ -215,19 +188,11 @@ function buildUserPrompt(
|
||||
documentText: string,
|
||||
chunkText: string,
|
||||
): string {
|
||||
// Tail-truncate `documentText` to `SYNOPSIS_DOC_MAX_CHARS` so small local
|
||||
// chat models don't stall on >100KB pages. Head preserved (title block,
|
||||
// frontmatter, intro paragraphs carry the document-level anchor).
|
||||
let trimmedDoc = documentText;
|
||||
if (documentText.length > SYNOPSIS_DOC_MAX_CHARS) {
|
||||
trimmedDoc = documentText.slice(0, SYNOPSIS_DOC_MAX_CHARS) +
|
||||
`\n\n[... ${documentText.length - SYNOPSIS_DOC_MAX_CHARS} chars truncated for synopsis budget ...]`;
|
||||
}
|
||||
return [
|
||||
`<page_title>${pageTitle}</page_title>`,
|
||||
'',
|
||||
'<full_document>',
|
||||
trimmedDoc,
|
||||
documentText,
|
||||
'</full_document>',
|
||||
'',
|
||||
'<chunk>',
|
||||
|
||||
+65
-73
@@ -40,6 +40,7 @@ import type {
|
||||
BrainStats, BrainHealth,
|
||||
IngestLogEntry, IngestLogInput,
|
||||
EngineConfig,
|
||||
ResolvedColumn,
|
||||
EvalCandidate, EvalCandidateInput,
|
||||
EvalCaptureFailure, EvalCaptureFailureReason,
|
||||
SalienceOpts, SalienceResult, AnomaliesOpts, AnomalyResult,
|
||||
@@ -57,8 +58,6 @@ import { finalizeLastSeen } from './chronicle/last-seen.ts';
|
||||
import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
import {
|
||||
normalizeEngineColumn,
|
||||
buildVectorCastFragment,
|
||||
@@ -2232,12 +2231,20 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Chunks
|
||||
async upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string } & BatchOpts): Promise<void> {
|
||||
async upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string; embeddingColumn?: ResolvedColumn } & BatchOpts): Promise<void> {
|
||||
return this.batchRetry(opts?.auditSite ?? 'upsertChunks', opts?.signal, () => this._upsertChunksOnce(slug, chunks, opts), chunks.length);
|
||||
}
|
||||
|
||||
private async _upsertChunksOnce(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string }): Promise<void> {
|
||||
private async _upsertChunksOnce(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string; embeddingColumn?: ResolvedColumn }): Promise<void> {
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// #1262: caller-resolved write target for TEXT embeddings. Descriptor
|
||||
// names are identifier-validated + quoted by buildVectorCastFragment;
|
||||
// omitted => legacy `embedding vector`. Mirrors postgres-engine.ts.
|
||||
const targetFragment = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn)
|
||||
: undefined;
|
||||
const targetCol = targetFragment?.col ?? 'embedding';
|
||||
const embeddingCast = targetFragment?.castSql.replace('$1::', '') ?? 'vector';
|
||||
|
||||
// Source-scope the page-id lookup so duplicate slugs in different sources
|
||||
// do not return multiple rows or target the wrong page.
|
||||
@@ -2272,7 +2279,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// list. Image chunks pass embedding=null + embedding_image=Float32Array
|
||||
// (1024-dim Voyage). Text/code chunks pass embedding=Float32Array +
|
||||
// embedding_image=null. Default modality='text' when omitted.
|
||||
const cols = '(page_id, chunk_index, chunk_text, chunk_source, embedding, model, token_count, embedded_at, language, symbol_name, symbol_type, start_line, end_line, parent_symbol_path, doc_comment, symbol_name_qualified, modality, embedding_image)';
|
||||
const cols = `(page_id, chunk_index, chunk_text, chunk_source, ${targetCol}, model, token_count, embedded_at, language, symbol_name, symbol_type, start_line, end_line, parent_symbol_path, doc_comment, symbol_name_qualified, modality, embedding_image)`;
|
||||
const rowParts: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let paramIdx = 1;
|
||||
@@ -2290,7 +2297,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const modality = chunk.modality ?? 'text';
|
||||
|
||||
// Inline ::vector NULL literals to avoid a per-branch placeholder.
|
||||
const embeddingPh = embeddingStr ? `$${paramIdx++}::vector` : 'NULL';
|
||||
const embeddingPh = embeddingStr ? `$${paramIdx++}::${embeddingCast}` : 'NULL';
|
||||
const embeddedAtPh = embeddingStr ? 'now()' : 'NULL';
|
||||
const embeddingImagePh = embeddingImageStr ? `$${paramIdx++}::vector` : 'NULL';
|
||||
|
||||
@@ -2324,41 +2331,37 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// v0.40.3.0 D24 NULL→non-NULL race fix mirrors postgres-engine.ts. Two writers
|
||||
// racing on the same chunk previously raced last-write-wins; the fix lets the
|
||||
// fresher `embedded_at` win in the text-unchanged branch.
|
||||
//
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
chunk_text = EXCLUDED.chunk_text,
|
||||
chunk_source = EXCLUDED.chunk_source,
|
||||
embedding = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.embedding
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.embedding
|
||||
${targetCol} = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.${targetCol}
|
||||
WHEN content_chunks.${targetCol} IS NULL THEN EXCLUDED.${targetCol}
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
THEN EXCLUDED.${targetCol}
|
||||
ELSE content_chunks.${targetCol}
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
WHEN content_chunks.embedding IS NULL AND EXCLUDED.embedding IS NOT NULL THEN EXCLUDED.embedded_at
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.${targetCol} IS NULL THEN NULL
|
||||
WHEN content_chunks.${targetCol} IS NULL AND EXCLUDED.${targetCol} IS NOT NULL THEN EXCLUDED.embedded_at
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.embedded_at
|
||||
ELSE content_chunks.embedded_at
|
||||
END,
|
||||
language = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.language ELSE COALESCE(EXCLUDED.language, content_chunks.language) END,
|
||||
symbol_name = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name ELSE COALESCE(EXCLUDED.symbol_name, content_chunks.symbol_name) END,
|
||||
symbol_type = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_type ELSE COALESCE(EXCLUDED.symbol_type, content_chunks.symbol_type) END,
|
||||
start_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.start_line ELSE COALESCE(EXCLUDED.start_line, content_chunks.start_line) END,
|
||||
end_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.end_line ELSE COALESCE(EXCLUDED.end_line, content_chunks.end_line) END,
|
||||
parent_symbol_path = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.parent_symbol_path ELSE COALESCE(EXCLUDED.parent_symbol_path, content_chunks.parent_symbol_path) END,
|
||||
doc_comment = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.doc_comment ELSE COALESCE(EXCLUDED.doc_comment, content_chunks.doc_comment) END,
|
||||
symbol_name_qualified = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name_qualified ELSE COALESCE(EXCLUDED.symbol_name_qualified, content_chunks.symbol_name_qualified) END,
|
||||
language = EXCLUDED.language,
|
||||
symbol_name = EXCLUDED.symbol_name,
|
||||
symbol_type = EXCLUDED.symbol_type,
|
||||
start_line = EXCLUDED.start_line,
|
||||
end_line = EXCLUDED.end_line,
|
||||
parent_symbol_path = EXCLUDED.parent_symbol_path,
|
||||
doc_comment = EXCLUDED.doc_comment,
|
||||
symbol_name_qualified = EXCLUDED.symbol_name_qualified,
|
||||
modality = EXCLUDED.modality,
|
||||
embedding_image = COALESCE(EXCLUDED.embedding_image, content_chunks.embedding_image)`,
|
||||
params
|
||||
@@ -2383,14 +2386,19 @@ export class PGLiteEngine implements BrainEngine {
|
||||
* drift (NULL grandfathered → never stale). Shared by countStaleChunks +
|
||||
* sumStaleChunkChars so they can't drift.
|
||||
*/
|
||||
private buildStaleChunkWhere(opts?: { sourceId?: string; signature?: string }): { where: string; params: unknown[] } {
|
||||
private buildStaleChunkWhere(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): { where: string; params: unknown[] } {
|
||||
// #1262: staleness targets the caller-resolved write column when set
|
||||
// (identifier-validated + quoted); legacy `embedding` otherwise.
|
||||
const staleCol = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn).col
|
||||
: 'embedding';
|
||||
const params: unknown[] = [];
|
||||
const conds: string[] = [];
|
||||
if (opts?.signature !== undefined) {
|
||||
params.push(opts.signature);
|
||||
conds.push(`(cc.embedding IS NULL OR (p.embedding_signature IS NOT NULL AND p.embedding_signature <> $${params.length}))`);
|
||||
conds.push(`(cc.${staleCol} IS NULL OR (p.embedding_signature IS NOT NULL AND p.embedding_signature <> $${params.length}))`);
|
||||
} else {
|
||||
conds.push(`cc.embedding IS NULL`);
|
||||
conds.push(`cc.${staleCol} IS NULL`);
|
||||
}
|
||||
conds.push(`NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')`);
|
||||
if (opts?.sourceId !== undefined) {
|
||||
@@ -2400,7 +2408,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return { where: conds.join(' AND '), params };
|
||||
}
|
||||
|
||||
async countStaleChunks(opts?: { sourceId?: string; signature?: string }): Promise<number> {
|
||||
async countStaleChunks(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number> {
|
||||
// D7: source-scoped count for `gbrain embed --stale --source X`. Always
|
||||
// JOIN pages so embed-skip + signature predicates apply. PGLite is
|
||||
// PostgreSQL 17.5 in WASM and supports the full JSONB operator set.
|
||||
@@ -2416,7 +2424,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return Number(count);
|
||||
}
|
||||
|
||||
async sumStaleChunkChars(opts?: { sourceId?: string; signature?: string }): Promise<number> {
|
||||
async sumStaleChunkChars(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number> {
|
||||
// Sibling of countStaleChunks: same stale predicate, summing chunk_text
|
||||
// length for the sync cost preview. ::bigint guards int4 overflow.
|
||||
const { where, params } = this.buildStaleChunkWhere(opts);
|
||||
@@ -2469,11 +2477,17 @@ export class PGLiteEngine implements BrainEngine {
|
||||
sourceId?: string;
|
||||
orderBy?: 'page_id' | 'updated_desc';
|
||||
afterUpdatedAt?: string | null;
|
||||
embeddingColumn?: ResolvedColumn;
|
||||
}): Promise<StaleChunkRow[]> {
|
||||
const limit = opts?.batchSize ?? 2000;
|
||||
const afterPid = opts?.afterPageId ?? 0;
|
||||
const afterIdx = opts?.afterChunkIndex ?? -1;
|
||||
const orderBy = opts?.orderBy ?? 'page_id';
|
||||
// #1262: staleness follows the caller-resolved write column (validated +
|
||||
// quoted identifier); legacy `embedding` otherwise.
|
||||
const staleCol = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn).col
|
||||
: 'embedding';
|
||||
|
||||
// v0.41.18.0 (A13, codex #9): --priority recent path. See postgres-engine
|
||||
// sibling for full rationale. Same composite cursor + ORDER BY.
|
||||
@@ -2487,7 +2501,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
ORDER BY p.updated_at DESC NULLS LAST, p.id ASC, cc.chunk_index ASC
|
||||
LIMIT $1`,
|
||||
@@ -2498,7 +2512,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (
|
||||
p.updated_at < $1::timestamptz
|
||||
@@ -2517,7 +2531,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND p.source_id = $1
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
ORDER BY p.updated_at DESC NULLS LAST, p.id ASC, cc.chunk_index ASC
|
||||
@@ -2529,7 +2543,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND p.source_id = $1
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (
|
||||
@@ -2554,7 +2568,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
cc.model, cc.token_count, p.source_id, cc.page_id
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (cc.page_id, cc.chunk_index) > ($1, $2)
|
||||
ORDER BY cc.page_id, cc.chunk_index
|
||||
@@ -2568,7 +2582,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
cc.model, cc.token_count, p.source_id, cc.page_id
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE cc.${staleCol} IS NULL
|
||||
AND p.source_id = $1
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (cc.page_id, cc.chunk_index) > ($2, $3)
|
||||
@@ -5208,31 +5222,26 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const { rows: [h] } = await this.db.query(`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
|
||||
),
|
||||
narrative_pages AS (
|
||||
-- Composition-aware score: code source files and calendar daily
|
||||
-- files are orphans-by-design (no inbound wikilinks, no Timeline
|
||||
-- fence). Excluding them from the orphan/link-density/timeline
|
||||
-- denominators keeps a bulk code/calendar import from cratering
|
||||
-- brain_score.
|
||||
SELECT id FROM pages WHERE type IS NULL OR type NOT IN ('code', 'calendar-index')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM narrative_pages) as narrative_page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
-- Bug 11 — orphan = islanded (no inbound AND no outbound). The raw
|
||||
-- list is filtered in TS using the shared orphan-reporting policy.
|
||||
0 as orphan_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
-- Bug 11 — orphan = islanded (no inbound AND no outbound).
|
||||
-- See BrainHealth.orphan_pages docstring; docs updated to match this.
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id)
|
||||
) as dead_links,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NULL) as missing_embeddings,
|
||||
(SELECT count(*) FROM links) as link_count,
|
||||
(SELECT count(DISTINCT te.page_id) FROM timeline_entries te
|
||||
WHERE te.page_id IN (SELECT id FROM narrative_pages)) as pages_with_timeline,
|
||||
(SELECT count(DISTINCT page_id) FROM timeline_entries) as pages_with_timeline,
|
||||
(SELECT count(*) FROM entity_pages e
|
||||
WHERE EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = e.id))::float /
|
||||
GREATEST((SELECT count(*) FROM entity_pages), 1)::float as link_coverage,
|
||||
@@ -5251,34 +5260,17 @@ export class PGLiteEngine implements BrainEngine {
|
||||
LIMIT 5
|
||||
`);
|
||||
|
||||
const { rows: islandedRows } = await this.db.query(`
|
||||
SELECT p.slug
|
||||
FROM pages p
|
||||
-- Narrative pages only (same type filter as the narrative_pages CTE):
|
||||
-- code/calendar-index pages are orphans-by-design and must not count
|
||||
-- against the noOrphans component (#1144).
|
||||
WHERE (p.type IS NULL OR p.type NOT IN ('code', 'calendar-index'))
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
`);
|
||||
|
||||
const r = h as Record<string, unknown>;
|
||||
const pageCount = Number(r.page_count);
|
||||
// Composition-aware denominators (excludes code/calendar-index pages);
|
||||
// a code-only brain has nothing narrative to penalize → full marks.
|
||||
const narrativePageCount = Number(r.narrative_page_count);
|
||||
const embedCoverage = Number(r.embed_coverage);
|
||||
const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
|
||||
const orphanOverrides = await loadOrphanPolicyOverrides(this);
|
||||
const orphanPages = (islandedRows as { slug: string }[])
|
||||
.filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length;
|
||||
const orphanPages = Number(r.orphan_pages);
|
||||
const deadLinks = Number(r.dead_links);
|
||||
const linkCount = Number(r.link_count);
|
||||
const pagesWithTimeline = Number(r.pages_with_timeline);
|
||||
|
||||
const linkDensity = narrativePageCount > 0 ? Math.min(linkCount / narrativePageCount, 1) : 1;
|
||||
const timelineCoverageDensity = narrativePageCount > 0 ? Math.min(pagesWithTimeline / narrativePageCount, 1) : 1;
|
||||
const noOrphans = narrativePageCount > 0 ? 1 - (orphanPages / narrativePageCount) : 1;
|
||||
const linkDensity = pageCount > 0 ? Math.min(linkCount / pageCount, 1) : 0;
|
||||
const timelineCoverageDensity = pageCount > 0 ? Math.min(pagesWithTimeline / pageCount, 1) : 0;
|
||||
const noOrphans = pageCount > 0 ? 1 - (orphanPages / pageCount) : 1;
|
||||
const noDeadLinks = pageCount > 0 ? 1 - Math.min(deadLinks / pageCount, 1) : 1;
|
||||
// Bug 11 — per-component points. Sum equals brainScore by construction
|
||||
// so `doctor` can render a breakdown that adds up to the total.
|
||||
@@ -5299,7 +5291,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return {
|
||||
page_count: pageCount,
|
||||
embed_coverage: embedCoverage,
|
||||
stale_pages: stalePages,
|
||||
stale_pages: Number(r.stale_pages),
|
||||
orphan_pages: orphanPages,
|
||||
missing_embeddings: Number(r.missing_embeddings),
|
||||
brain_score: brainScore,
|
||||
|
||||
+69
-76
@@ -50,6 +50,7 @@ import type {
|
||||
BrainStats, BrainHealth,
|
||||
IngestLogEntry, IngestLogInput,
|
||||
EngineConfig,
|
||||
ResolvedColumn,
|
||||
EvalCandidate, EvalCandidateInput,
|
||||
EvalCaptureFailure, EvalCaptureFailureReason,
|
||||
SalienceOpts, SalienceResult, AnomaliesOpts, AnomalyResult,
|
||||
@@ -67,8 +68,6 @@ import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts';
|
||||
import { DELETE_BATCH_SIZE } from './engine-constants.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
|
||||
function escapeSqlStringLiteral(value: string): string {
|
||||
return value.replace(/'/g, "''");
|
||||
@@ -2382,13 +2381,21 @@ export class PostgresEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
// Chunks
|
||||
async upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string } & BatchOpts): Promise<void> {
|
||||
async upsertChunks(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string; embeddingColumn?: ResolvedColumn } & BatchOpts): Promise<void> {
|
||||
return this.batchRetry(opts?.auditSite ?? 'upsertChunks', opts?.signal, () => this._upsertChunksOnce(slug, chunks, opts), chunks.length);
|
||||
}
|
||||
|
||||
private async _upsertChunksOnce(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string }): Promise<void> {
|
||||
private async _upsertChunksOnce(slug: string, chunks: ChunkInput[], opts?: { sourceId?: string; embeddingColumn?: ResolvedColumn }): Promise<void> {
|
||||
const sql = this.sql;
|
||||
const sourceId = opts?.sourceId ?? 'default';
|
||||
// #1262: caller-resolved write target for TEXT embeddings. Descriptor
|
||||
// names are identifier-validated + quoted by buildVectorCastFragment;
|
||||
// omitted => legacy `embedding vector`.
|
||||
const targetFragment = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn)
|
||||
: undefined;
|
||||
const targetCol = targetFragment?.col ?? 'embedding';
|
||||
const embeddingCast = targetFragment?.castSql.replace('$1::', '') ?? 'vector';
|
||||
|
||||
// Source-scope the page-id lookup. Without this filter, multi-source
|
||||
// brains where the slug exists in 2+ sources return >1 row and the
|
||||
@@ -2415,7 +2422,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
// scope metadata through upserts.
|
||||
// v0.27.1 (Phase 8): added `modality` + `embedding_image` to the column
|
||||
// list. Image chunks pass embedding=null + embedding_image=Float32Array.
|
||||
const cols = '(page_id, chunk_index, chunk_text, chunk_source, embedding, model, token_count, embedded_at, language, symbol_name, symbol_type, start_line, end_line, parent_symbol_path, doc_comment, symbol_name_qualified, modality, embedding_image)';
|
||||
const cols = `(page_id, chunk_index, chunk_text, chunk_source, ${targetCol}, model, token_count, embedded_at, language, symbol_name, symbol_type, start_line, end_line, parent_symbol_path, doc_comment, symbol_name_qualified, modality, embedding_image)`;
|
||||
const rows: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let paramIdx = 1;
|
||||
@@ -2432,7 +2439,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
: null;
|
||||
const modality = chunk.modality ?? 'text';
|
||||
|
||||
const embeddingPh = embeddingStr ? `$${paramIdx++}::vector` : 'NULL';
|
||||
const embeddingPh = embeddingStr ? `$${paramIdx++}::${embeddingCast}` : 'NULL';
|
||||
const embeddedAtPh = embeddingStr ? 'now()' : 'NULL';
|
||||
const embeddingImagePh = embeddingImageStr ? `$${paramIdx++}::vector` : 'NULL';
|
||||
|
||||
@@ -2475,44 +2482,37 @@ export class PostgresEngine implements BrainEngine {
|
||||
// - new is fresher (embedded_at > existing.embedded_at) → take new
|
||||
// - otherwise → keep existing (slower writer with stale embedding loses)
|
||||
// Mirrored in pglite-engine.ts; pinned by test/e2e/concurrent-embed-race.test.ts.
|
||||
//
|
||||
// Code-chunk metadata columns (language / symbol_name / symbol_type / line range /
|
||||
// parent_symbol_path / doc_comment / symbol_name_qualified) follow the SAME chunk_text-gated
|
||||
// CASE pattern as `embedding` (#769). Re-chunk (chunk_text changed) trusts EXCLUDED outright;
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
chunk_text = EXCLUDED.chunk_text,
|
||||
chunk_source = EXCLUDED.chunk_source,
|
||||
embedding = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.embedding
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.embedding
|
||||
${targetCol} = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.${targetCol}
|
||||
WHEN content_chunks.${targetCol} IS NULL THEN EXCLUDED.${targetCol}
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
THEN EXCLUDED.${targetCol}
|
||||
ELSE content_chunks.${targetCol}
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
WHEN content_chunks.embedding IS NULL AND EXCLUDED.embedding IS NOT NULL THEN EXCLUDED.embedded_at
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.${targetCol} IS NULL THEN NULL
|
||||
WHEN content_chunks.${targetCol} IS NULL AND EXCLUDED.${targetCol} IS NOT NULL THEN EXCLUDED.embedded_at
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.embedded_at
|
||||
ELSE content_chunks.embedded_at
|
||||
END,
|
||||
language = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.language ELSE COALESCE(EXCLUDED.language, content_chunks.language) END,
|
||||
symbol_name = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name ELSE COALESCE(EXCLUDED.symbol_name, content_chunks.symbol_name) END,
|
||||
symbol_type = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_type ELSE COALESCE(EXCLUDED.symbol_type, content_chunks.symbol_type) END,
|
||||
start_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.start_line ELSE COALESCE(EXCLUDED.start_line, content_chunks.start_line) END,
|
||||
end_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.end_line ELSE COALESCE(EXCLUDED.end_line, content_chunks.end_line) END,
|
||||
parent_symbol_path = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.parent_symbol_path ELSE COALESCE(EXCLUDED.parent_symbol_path, content_chunks.parent_symbol_path) END,
|
||||
doc_comment = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.doc_comment ELSE COALESCE(EXCLUDED.doc_comment, content_chunks.doc_comment) END,
|
||||
symbol_name_qualified = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name_qualified ELSE COALESCE(EXCLUDED.symbol_name_qualified, content_chunks.symbol_name_qualified) END,
|
||||
language = EXCLUDED.language,
|
||||
symbol_name = EXCLUDED.symbol_name,
|
||||
symbol_type = EXCLUDED.symbol_type,
|
||||
start_line = EXCLUDED.start_line,
|
||||
end_line = EXCLUDED.end_line,
|
||||
parent_symbol_path = EXCLUDED.parent_symbol_path,
|
||||
doc_comment = EXCLUDED.doc_comment,
|
||||
symbol_name_qualified = EXCLUDED.symbol_name_qualified,
|
||||
modality = EXCLUDED.modality,
|
||||
embedding_image = COALESCE(EXCLUDED.embedding_image, content_chunks.embedding_image)`,
|
||||
params as Parameters<typeof sql.unsafe>[1],
|
||||
@@ -2539,14 +2539,19 @@ export class PostgresEngine implements BrainEngine {
|
||||
* embedding_signature drift (NULL grandfathered). Shared by
|
||||
* countStaleChunks + sumStaleChunkChars (parity with the PGLite sibling).
|
||||
*/
|
||||
private buildStaleChunkWhere(opts?: { sourceId?: string; signature?: string }): { where: string; params: unknown[] } {
|
||||
private buildStaleChunkWhere(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): { where: string; params: unknown[] } {
|
||||
// #1262: staleness targets the caller-resolved write column when set
|
||||
// (identifier-validated + quoted); legacy `embedding` otherwise.
|
||||
const staleCol = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn).col
|
||||
: 'embedding';
|
||||
const params: unknown[] = [];
|
||||
const conds: string[] = [];
|
||||
if (opts?.signature !== undefined) {
|
||||
params.push(opts.signature);
|
||||
conds.push(`(cc.embedding IS NULL OR (p.embedding_signature IS NOT NULL AND p.embedding_signature <> $${params.length}))`);
|
||||
conds.push(`(cc.${staleCol} IS NULL OR (p.embedding_signature IS NOT NULL AND p.embedding_signature <> $${params.length}))`);
|
||||
} else {
|
||||
conds.push(`cc.embedding IS NULL`);
|
||||
conds.push(`cc.${staleCol} IS NULL`);
|
||||
}
|
||||
conds.push(`NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')`);
|
||||
if (opts?.sourceId !== undefined) {
|
||||
@@ -2556,7 +2561,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
return { where: conds.join(' AND '), params };
|
||||
}
|
||||
|
||||
async countStaleChunks(opts?: { sourceId?: string; signature?: string }): Promise<number> {
|
||||
async countStaleChunks(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number> {
|
||||
// Always JOIN pages so the embed_skip + signature predicates apply.
|
||||
// D7: source_id scoping. v0.41.31: optional signature widens staleness
|
||||
// to embedding_signature drift (NULL grandfathered).
|
||||
@@ -2574,7 +2579,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
});
|
||||
}
|
||||
|
||||
async sumStaleChunkChars(opts?: { sourceId?: string; signature?: string }): Promise<number> {
|
||||
async sumStaleChunkChars(opts?: { sourceId?: string; signature?: string; embeddingColumn?: ResolvedColumn }): Promise<number> {
|
||||
// Sibling of countStaleChunks: same stale predicate, summing chunk_text
|
||||
// length for the sync cost preview. ::bigint guards int4 overflow.
|
||||
const { where, params } = this.buildStaleChunkWhere(opts);
|
||||
@@ -2627,11 +2632,18 @@ export class PostgresEngine implements BrainEngine {
|
||||
sourceId?: string;
|
||||
orderBy?: 'page_id' | 'updated_desc';
|
||||
afterUpdatedAt?: string | null;
|
||||
embeddingColumn?: ResolvedColumn;
|
||||
}): Promise<StaleChunkRow[]> {
|
||||
const limit = opts?.batchSize ?? 2000;
|
||||
const afterPid = opts?.afterPageId ?? 0;
|
||||
const afterIdx = opts?.afterChunkIndex ?? -1;
|
||||
const orderBy = opts?.orderBy ?? 'page_id';
|
||||
// #1262: staleness follows the caller-resolved write column (validated +
|
||||
// quoted identifier); legacy `embedding` otherwise. Interpolated below as
|
||||
// an unsafe FRAGMENT (identifiers can't be bound parameters).
|
||||
const staleCol = opts?.embeddingColumn
|
||||
? buildVectorCastFragment(opts.embeddingColumn).col
|
||||
: 'embedding';
|
||||
|
||||
// RLS scope binding (opt-in via GBRAIN_RLS_SCOPE_BINDING).
|
||||
return await this.withScopedReadTransaction(undefined, opts?.sourceId, async (tx) => {
|
||||
@@ -2648,7 +2660,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
ORDER BY p.updated_at DESC NULLS LAST, p.id ASC, cc.chunk_index ASC
|
||||
LIMIT ${limit}
|
||||
@@ -2658,7 +2670,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (
|
||||
p.updated_at < ${afterUpdated}::timestamptz
|
||||
@@ -2676,7 +2688,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND p.source_id = ${opts.sourceId}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
ORDER BY p.updated_at DESC NULLS LAST, p.id ASC, cc.chunk_index ASC
|
||||
@@ -2687,7 +2699,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
p.updated_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND p.source_id = ${opts.sourceId}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (
|
||||
@@ -2707,7 +2719,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
cc.model, cc.token_count, p.source_id, cc.page_id
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (cc.page_id, cc.chunk_index) > (${afterPid}, ${afterIdx})
|
||||
ORDER BY cc.page_id, cc.chunk_index
|
||||
@@ -2720,7 +2732,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
cc.model, cc.token_count, p.source_id, cc.page_id
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NULL
|
||||
WHERE ${tx.unsafe(`cc.${staleCol} IS NULL`)}
|
||||
AND p.source_id = ${opts.sourceId}
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')
|
||||
AND (cc.page_id, cc.chunk_index) > (${afterPid}, ${afterIdx})
|
||||
@@ -5322,35 +5334,32 @@ export class PostgresEngine implements BrainEngine {
|
||||
async getHealth(): Promise<BrainHealth> {
|
||||
const sql = this.sql;
|
||||
// Bug 11 doc-drift fix — orphan_pages means "islanded" (no inbound AND
|
||||
// no outbound links). The raw islanded list is filtered through the same
|
||||
// policy as `gbrain orphans` so convention pages do not count against
|
||||
// dashboard health.
|
||||
// no outbound links), aligning both engines with the user-facing
|
||||
// definition. The type comment previously said "no inbound" but the
|
||||
// SQL required both — docs now match code so users can trust the
|
||||
// number. A hub page that links out to many but has no back-references
|
||||
// is working as intended, not an orphan.
|
||||
const [h] = await sql`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
|
||||
),
|
||||
narrative_pages AS (
|
||||
-- Composition-aware score: code source files and calendar daily
|
||||
-- files are orphans-by-design (no inbound wikilinks, no Timeline
|
||||
-- fence). Excluding them from the orphan/link-density/timeline
|
||||
-- denominators keeps a bulk code/calendar import from cratering
|
||||
-- brain_score.
|
||||
SELECT id FROM pages WHERE type IS NULL OR type NOT IN ('code', 'calendar-index')
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM narrative_pages) as narrative_page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
0 as orphan_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id)
|
||||
) as dead_links,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NULL) as missing_embeddings,
|
||||
(SELECT count(*) FROM links) as link_count,
|
||||
(SELECT count(DISTINCT te.page_id) FROM timeline_entries te
|
||||
WHERE te.page_id IN (SELECT id FROM narrative_pages)) as pages_with_timeline,
|
||||
(SELECT count(DISTINCT page_id) FROM timeline_entries) as pages_with_timeline,
|
||||
(SELECT count(*) FROM entity_pages e
|
||||
WHERE EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = e.id))::float /
|
||||
GREATEST((SELECT count(*) FROM entity_pages), 1)::float as link_coverage,
|
||||
@@ -5368,33 +5377,17 @@ export class PostgresEngine implements BrainEngine {
|
||||
LIMIT 5
|
||||
`;
|
||||
|
||||
const islandedRows = await sql<{ slug: string }[]>`
|
||||
SELECT p.slug
|
||||
FROM pages p
|
||||
-- Narrative pages only (same type filter as the narrative_pages CTE):
|
||||
-- code/calendar-index pages are orphans-by-design and must not count
|
||||
-- against the noOrphans component (#1144).
|
||||
WHERE (p.type IS NULL OR p.type NOT IN ('code', 'calendar-index'))
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
`;
|
||||
|
||||
const pageCount = Number(h.page_count);
|
||||
// Composition-aware denominators (excludes code/calendar-index pages);
|
||||
// a code-only brain has nothing narrative to penalize → full marks.
|
||||
const narrativePageCount = Number(h.narrative_page_count);
|
||||
const embedCoverage = Number(h.embed_coverage);
|
||||
const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
|
||||
const orphanOverrides = await loadOrphanPolicyOverrides(this);
|
||||
const orphanPages = islandedRows.filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length;
|
||||
const orphanPages = Number(h.orphan_pages);
|
||||
const deadLinks = Number(h.dead_links);
|
||||
const linkCount = Number(h.link_count);
|
||||
const pagesWithTimeline = Number(h.pages_with_timeline);
|
||||
|
||||
// brain_score: 0-100 weighted average
|
||||
const linkDensity = narrativePageCount > 0 ? Math.min(linkCount / narrativePageCount, 1) : 1;
|
||||
const timelineCoverageWhole = narrativePageCount > 0 ? Math.min(pagesWithTimeline / narrativePageCount, 1) : 1;
|
||||
const noOrphans = narrativePageCount > 0 ? 1 - (orphanPages / narrativePageCount) : 1;
|
||||
const linkDensity = pageCount > 0 ? Math.min(linkCount / pageCount, 1) : 0;
|
||||
const timelineCoverageWhole = pageCount > 0 ? Math.min(pagesWithTimeline / pageCount, 1) : 0;
|
||||
const noOrphans = pageCount > 0 ? 1 - (orphanPages / pageCount) : 1;
|
||||
const noDeadLinks = pageCount > 0 ? 1 - Math.min(deadLinks / pageCount, 1) : 1;
|
||||
// Per-component points. Sum equals brainScore by construction.
|
||||
//
|
||||
@@ -5414,7 +5407,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
return {
|
||||
page_count: pageCount,
|
||||
embed_coverage: embedCoverage,
|
||||
stale_pages: stalePages,
|
||||
stale_pages: Number(h.stale_pages),
|
||||
orphan_pages: orphanPages,
|
||||
missing_embeddings: Number(h.missing_embeddings),
|
||||
brain_score: brainScore,
|
||||
|
||||
@@ -443,6 +443,80 @@ export function resolveEmbeddingColumn(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the WRITE-side embedding column for the currently configured
|
||||
* embedding model (#1262). The read-side resolver above answers "which
|
||||
* column does this query search?"; this one answers "which column should
|
||||
* newly produced text embeddings land in?".
|
||||
*
|
||||
* Unlike read-side search, writes take no per-call column override. The
|
||||
* import/embed boundary resolves once from merged config + gateway state
|
||||
* and passes the descriptor into `engine.upsertChunks`; engines stay
|
||||
* config-free (same contract as the read-side descriptor).
|
||||
*
|
||||
* Behavior:
|
||||
* - no user-declared `embedding_columns` => undefined (legacy brain,
|
||||
* writes keep targeting the default `embedding` column)
|
||||
* - a user-declared entry whose `provider` matches the current
|
||||
* embedding model => that entry's descriptor
|
||||
* - no provider match => undefined (fall back to legacy `embedding`)
|
||||
*
|
||||
* Only USER-declared entries are consulted — never the cfg-derived
|
||||
* builtins. The `embedding_image` builtin's provider is the multimodal
|
||||
* model; matching it here would misroute text embeddings into the image
|
||||
* column. The no-match fallback is intentional: switching models before
|
||||
* registering a matching column must not silently write vectors into an
|
||||
* arbitrary column.
|
||||
*/
|
||||
export function resolveWriteColumn(cfg: GBrainConfig): ResolvedColumn | undefined {
|
||||
const userColumns = cfg.embedding_columns;
|
||||
if (
|
||||
!userColumns ||
|
||||
typeof userColumns !== 'object' ||
|
||||
Array.isArray(userColumns) ||
|
||||
Object.keys(userColumns).length === 0
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Same model-resolution chain as the registry builtin: cfg > gateway > default.
|
||||
let gwModel: string | undefined;
|
||||
try {
|
||||
const gw = require('../ai/gateway.ts') as typeof import('../ai/gateway.ts');
|
||||
gwModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
// Gateway unconfigured — fall through to the canonical default.
|
||||
}
|
||||
const currentModel = cfg.embedding_model ?? gwModel ?? DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const [name, entry] of Object.entries(userColumns)) {
|
||||
if (!entry) continue;
|
||||
validateColumnKey(name);
|
||||
validateColumnConfig(name, entry);
|
||||
if (entry.provider !== currentModel) continue;
|
||||
return {
|
||||
name,
|
||||
type: entry.type,
|
||||
dimensions: entry.dimensions,
|
||||
embeddingModel: entry.provider,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Engine-boundary convenience: merged config (file/env + DB plane) →
|
||||
* resolveWriteColumn. Dynamic import keeps config.ts out of this module's
|
||||
* static graph (mirrors the gateway require above).
|
||||
*/
|
||||
export async function resolveWriteColumnForEngine(
|
||||
engine: { getConfig(key: string): Promise<string | null | undefined> },
|
||||
): Promise<ResolvedColumn | undefined> {
|
||||
const { loadConfigWithEngine } = await import('../config.ts');
|
||||
const cfg = await loadConfigWithEngine(engine);
|
||||
return cfg ? resolveWriteColumn(cfg) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the resolved column is the default `embedding` name.
|
||||
* Name-based check; does not compare embedding space.
|
||||
|
||||
@@ -93,13 +93,7 @@ import { resolveLrSchedule } from './lr-schedule.ts';
|
||||
import { preflight, formatPreflightReport } from './preflight.ts';
|
||||
import { isRejected, loadRejectedBuffer, makeRejectedEntry, saveRejectedBuffer } from './rejected-buffer.ts';
|
||||
import { runReflect, runOneShotRewrite, describeJudges } from './reflect.ts';
|
||||
import {
|
||||
acceptCandidate,
|
||||
proposedPath as proposedFilePath,
|
||||
revertAllPending,
|
||||
skillPath,
|
||||
writeProposed,
|
||||
} from './version-store.ts';
|
||||
import { acceptCandidate, bestPath, revertAllPending, skillPath, writeProposed } from './version-store.ts';
|
||||
import { runValidationGate, scoreSkillOnTasks } from './validate-gate.ts';
|
||||
import { ROLLOUT_SUCCESS_THRESHOLD } from './types.ts';
|
||||
import type { SkillOptOpts, EditOp, RunReceipt, BenchmarkTask } from './types.ts';
|
||||
@@ -708,9 +702,9 @@ async function runOptimizationLoop(
|
||||
// to the catch's assignment values only (it can't prove the async callback ran).
|
||||
const finalOutcome = outcome as 'accepted' | 'no_improvement' | 'aborted' | 'errored';
|
||||
if (!mutateDecision.mutate && finalOutcome === 'accepted') {
|
||||
// writeProposed() emitted both the best pointer and the stable review
|
||||
// artifact in the accept branch. SKILL.md remains untouched.
|
||||
proposedPath = proposedFilePath(skillsDir, skillName);
|
||||
// best.md was written by writeProposed() in the accept branch (no-mutate
|
||||
// path); it doubles as proposed.md for human review. SKILL.md untouched.
|
||||
proposedPath = bestPath(skillsDir, skillName);
|
||||
} else if (mutateDecision.mutate) {
|
||||
mutatedSkillFile = finalOutcome === 'accepted';
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
*
|
||||
* history.json
|
||||
* best.md
|
||||
* proposed.md
|
||||
* versions/
|
||||
* v0001_e1_s1.md
|
||||
* v0002_e1_s2.md
|
||||
@@ -53,10 +52,6 @@ export function bestPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skilloptDir(skillsDir, skillName), 'best.md');
|
||||
}
|
||||
|
||||
export function proposedPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skilloptDir(skillsDir, skillName), 'proposed.md');
|
||||
}
|
||||
|
||||
export function skillPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skillsDir, skillName, 'SKILL.md');
|
||||
}
|
||||
@@ -176,18 +171,17 @@ export function acceptCandidate(input: AcceptInput): AcceptResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the candidate to both `best.md` and `proposed.md` WITHOUT touching
|
||||
* SKILL.md or the history ledger. `best.md` remains the optimizer's current
|
||||
* best pointer; `proposed.md` is the stable human-review artifact promised by
|
||||
* `--no-mutate`. Returns the proposal path. Each write is atomic (.tmp + rename).
|
||||
* Write the candidate to `best.md` (which doubles as `proposed.md`) WITHOUT
|
||||
* touching SKILL.md or the history ledger. Used by the `--no-mutate` /
|
||||
* bundled-without-allow paths: the optimizer found a better candidate but the
|
||||
* caller opted out of in-place mutation, so we surface it for human review.
|
||||
* Returns the path written. Atomic (.tmp + rename).
|
||||
*/
|
||||
export function writeProposed(skillsDir: string, skillName: string, candidateText: string): string {
|
||||
const best = bestPath(skillsDir, skillName);
|
||||
const proposed = proposedPath(skillsDir, skillName);
|
||||
fs.mkdirSync(path.dirname(best), { recursive: true });
|
||||
atomicWrite(best, candidateText);
|
||||
atomicWrite(proposed, candidateText);
|
||||
return proposed;
|
||||
const p = bestPath(skillsDir, skillName);
|
||||
fs.mkdirSync(path.dirname(p), { recursive: true });
|
||||
atomicWrite(p, candidateText);
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -63,26 +63,25 @@ interface PluginCtx {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export function register(api: PluginApi) {
|
||||
api.registerContextEngine(ENGINE_ID, (ctx: PluginCtx) => {
|
||||
const hostResolver =
|
||||
typeof ctx.resolveEntities === 'function'
|
||||
? ctx.resolveEntities
|
||||
: typeof ctx.brainQuery === 'function'
|
||||
? ctx.brainQuery
|
||||
: undefined;
|
||||
return createGBrainContextEngine({
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
resolveEntities: hostResolver,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const entry: PluginEntry = {
|
||||
id: 'gbrain-context-engine',
|
||||
name: 'GBrain Context Engine',
|
||||
description: 'Deterministic temporal/spatial context injection on every turn',
|
||||
register,
|
||||
|
||||
register(api: PluginApi) {
|
||||
api.registerContextEngine(ENGINE_ID, (ctx: PluginCtx) => {
|
||||
const hostResolver =
|
||||
typeof ctx.resolveEntities === 'function'
|
||||
? ctx.resolveEntities
|
||||
: typeof ctx.brainQuery === 'function'
|
||||
? ctx.brainQuery
|
||||
: undefined;
|
||||
return createGBrainContextEngine({
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
resolveEntities: hostResolver,
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export default entry;
|
||||
|
||||
@@ -119,40 +119,6 @@ describe('Bug 11 — orphan_pages is "no inbound links"', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('#1144 — brain_score is composition-aware', () => {
|
||||
test('code and calendar-index pages do not count as orphans or dilute density metrics', async () => {
|
||||
// Two linked narrative pages + a flood of orphan-by-design pages.
|
||||
await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: 'a', frontmatter: {} });
|
||||
await engine.putPage('people/bob', { type: 'person', title: 'Bob', compiled_truth: 'b', frontmatter: {} });
|
||||
const aId = (await (engine as any).db.query(`SELECT id FROM pages WHERE slug='people/alice'`)).rows[0].id;
|
||||
const bId = (await (engine as any).db.query(`SELECT id FROM pages WHERE slug='people/bob'`)).rows[0].id;
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO links (from_page_id, to_page_id, link_type) VALUES ($1, $2, 'mentions')`,
|
||||
[aId, bId],
|
||||
);
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await engine.putPage(`code/src/f${i}.py`, { type: 'code', title: `f${i}.py`, compiled_truth: 'def f(): pass', frontmatter: {} });
|
||||
}
|
||||
await engine.putPage('daily/calendar/2026-01-01', { type: 'calendar-index', title: '2026-01-01', compiled_truth: 'events', frontmatter: {} });
|
||||
|
||||
const h = await engine.getHealth();
|
||||
// 11 unlinked code/calendar pages exist, but both narrative pages are
|
||||
// linked → 0 orphans, full no-orphans marks.
|
||||
expect(h.orphan_pages).toBe(0);
|
||||
expect(h.no_orphans_score).toBe(15);
|
||||
// Link density: 1 link / 2 narrative pages, not 1 / 13 total pages.
|
||||
expect(h.link_density_score).toBe(Math.round(0.5 * 25));
|
||||
});
|
||||
|
||||
test('a brain with ONLY non-narrative pages gets full composition marks', async () => {
|
||||
await engine.putPage('code/src/only.py', { type: 'code', title: 'only.py', compiled_truth: 'x = 1', frontmatter: {} });
|
||||
const h = await engine.getHealth();
|
||||
expect(h.no_orphans_score).toBe(15);
|
||||
expect(h.link_density_score).toBe(25);
|
||||
expect(h.timeline_coverage_score).toBe(15);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Bug 11 — doctor renders brain_score breakdown', () => {
|
||||
test('doctor source contains brain_score breakdown rendering', async () => {
|
||||
const source = await Bun.file(new URL('../src/commands/doctor.ts', import.meta.url)).text();
|
||||
|
||||
@@ -241,3 +241,136 @@ describe('buildVectorCastFragment — engine SQL composer (D3)', () => {
|
||||
expect(castSql).toBe('$1::halfvec(2560)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('PGLite engine: upsertChunks write-side ResolvedColumn descriptor (#1262)', () => {
|
||||
test('halfvec descriptor writes the text embedding to the alternate column, not legacy embedding', async () => {
|
||||
await engine.putPage('docs/write-alt-pglite', {
|
||||
type: 'concept',
|
||||
title: 'Write alt column PGLite',
|
||||
compiled_truth: 'PGLite write-side alternate embedding column test.',
|
||||
});
|
||||
|
||||
const descriptor: ResolvedColumn = {
|
||||
name: 'embedding_ze',
|
||||
type: 'halfvec',
|
||||
dimensions: 2560,
|
||||
embeddingModel: 'zeroentropyai:zembed-1',
|
||||
};
|
||||
await engine.upsertChunks('docs/write-alt-pglite', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'PGLite write-side alternate embedding column test.',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(2560).fill(0.25),
|
||||
},
|
||||
], { embeddingColumn: descriptor });
|
||||
|
||||
const rows = await engine.executeRaw<{
|
||||
has_default: boolean;
|
||||
has_ze: boolean;
|
||||
has_embedded_at: boolean;
|
||||
}>(
|
||||
`SELECT embedding IS NOT NULL AS has_default,
|
||||
embedding_ze IS NOT NULL AS has_ze,
|
||||
embedded_at IS NOT NULL AS has_embedded_at
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/write-alt-pglite'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].has_default).toBe(false);
|
||||
expect(rows[0].has_ze).toBe(true);
|
||||
expect(rows[0].has_embedded_at).toBe(true);
|
||||
});
|
||||
|
||||
test('text-unchanged re-upsert without a vector preserves the alternate-column embedding', async () => {
|
||||
const descriptor: ResolvedColumn = {
|
||||
name: 'embedding_ze',
|
||||
type: 'halfvec',
|
||||
dimensions: 2560,
|
||||
embeddingModel: 'zeroentropyai:zembed-1',
|
||||
};
|
||||
// Same chunk_text, no embedding: the ON CONFLICT CASE must keep the
|
||||
// existing alternate-column vector (D24 semantics follow the column).
|
||||
await engine.upsertChunks('docs/write-alt-pglite', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'PGLite write-side alternate embedding column test.',
|
||||
chunk_source: 'compiled_truth',
|
||||
},
|
||||
], { embeddingColumn: descriptor });
|
||||
const rows = await engine.executeRaw<{ has_ze: boolean }>(
|
||||
`SELECT embedding_ze IS NOT NULL AS has_ze
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/write-alt-pglite'`,
|
||||
);
|
||||
expect(rows).toEqual([{ has_ze: true }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PGLite: embed --stale converges on an alt-column brain (#1262)', () => {
|
||||
test('boundary resolves the write column; stale scan does not re-select embedded rows', async () => {
|
||||
const { runEmbedCore } = await import('../../src/commands/embed.ts');
|
||||
const local = new PGLiteEngine();
|
||||
const previousHome = process.env.GBRAIN_HOME;
|
||||
process.env.GBRAIN_HOME = `/tmp/gbrain-write-col-stale-${Date.now()}`;
|
||||
try {
|
||||
await local.connect({});
|
||||
await local.initSchema();
|
||||
await (local as any).db.exec(
|
||||
`ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS embedding_ze halfvec(2560)`,
|
||||
);
|
||||
|
||||
const descriptor: ResolvedColumn = {
|
||||
name: 'embedding_ze',
|
||||
type: 'halfvec',
|
||||
dimensions: 2560,
|
||||
embeddingModel: 'zeroentropyai:zembed-1',
|
||||
};
|
||||
await local.setConfig('embedding_columns', JSON.stringify({
|
||||
embedding_ze: { provider: 'zeroentropyai:zembed-1', dimensions: 2560, type: 'halfvec' },
|
||||
}));
|
||||
configureGateway({
|
||||
embedding_model: 'zeroentropyai:zembed-1',
|
||||
embedding_dimensions: 2560,
|
||||
env: {},
|
||||
});
|
||||
|
||||
await local.putPage('docs/stale-alt-pglite', {
|
||||
type: 'concept',
|
||||
title: 'Dynamic stale column',
|
||||
compiled_truth: 'A chunk that is embedded only in the dynamic column.',
|
||||
});
|
||||
await local.upsertChunks('docs/stale-alt-pglite', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'A chunk that is embedded only in the dynamic column.',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(2560).fill(0.25),
|
||||
},
|
||||
], { embeddingColumn: descriptor });
|
||||
|
||||
// Engine-level contrast: legacy predicate still sees the row as stale;
|
||||
// the alt-column predicate does not.
|
||||
expect(await local.countStaleChunks()).toBe(1);
|
||||
expect(await local.countStaleChunks({ embeddingColumn: descriptor })).toBe(0);
|
||||
// sumStaleChunkChars feeds the sync cost gate — same predicate contract.
|
||||
expect(await local.sumStaleChunkChars()).toBeGreaterThan(0);
|
||||
expect(await local.sumStaleChunkChars({ embeddingColumn: descriptor })).toBe(0);
|
||||
expect(await local.listStaleChunks({ embeddingColumn: descriptor, batchSize: 100 })).toHaveLength(0);
|
||||
expect(await local.listStaleChunks({ batchSize: 100 })).toHaveLength(1);
|
||||
|
||||
// Boundary-level: `embed --stale --dry-run` resolves the write column
|
||||
// from merged config + gateway and reports NOTHING to embed. Without
|
||||
// the fix this reports 1 (perpetual re-embed loop).
|
||||
const result = await runEmbedCore(local, { stale: true, dryRun: true });
|
||||
expect(result.would_embed).toBe(0);
|
||||
} finally {
|
||||
await local.disconnect();
|
||||
if (previousHome === undefined) delete process.env.GBRAIN_HOME;
|
||||
else process.env.GBRAIN_HOME = previousHome;
|
||||
resetGateway();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -224,4 +224,54 @@ if (!dbUrl) {
|
||||
await engine.executeRaw(`UPDATE content_chunks SET embedding_voyage = '${v}'::vector WHERE id = ${dogId}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Postgres: upsertChunks write-side ResolvedColumn descriptor (#1262)', () => {
|
||||
const descriptor: ResolvedColumn = {
|
||||
name: 'embedding_ze',
|
||||
type: 'halfvec',
|
||||
dimensions: 2560,
|
||||
embeddingModel: 'zeroentropyai:zembed-1',
|
||||
};
|
||||
|
||||
test('halfvec descriptor writes the text embedding to the alternate column, not legacy embedding', async () => {
|
||||
await engine.putPage('docs/write-alt-postgres', {
|
||||
type: 'concept',
|
||||
title: 'Write alt column Postgres',
|
||||
compiled_truth: 'Postgres write-side alternate embedding column test.',
|
||||
});
|
||||
await engine.upsertChunks('docs/write-alt-postgres', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'Postgres write-side alternate embedding column test.',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(2560).fill(0.25),
|
||||
},
|
||||
], { embeddingColumn: descriptor });
|
||||
|
||||
const rows = await engine.executeRaw<{
|
||||
has_default: boolean;
|
||||
has_ze: boolean;
|
||||
}>(
|
||||
`SELECT embedding IS NOT NULL AS has_default,
|
||||
embedding_ze IS NOT NULL AS has_ze
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/write-alt-postgres'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].has_default).toBe(false);
|
||||
expect(rows[0].has_ze).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('stale scan follows the write-side column (count + list parity with the write target)', async () => {
|
||||
// Legacy predicate: cat/dog/write-alt rows all have embedding NULL.
|
||||
expect(await engine.countStaleChunks()).toBeGreaterThan(0);
|
||||
// Alt-column predicate: every chunk has embedding_ze populated.
|
||||
expect(await engine.countStaleChunks({ embeddingColumn: descriptor })).toBe(0);
|
||||
expect(await engine.listStaleChunks({ embeddingColumn: descriptor, batchSize: 100 })).toHaveLength(0);
|
||||
expect((await engine.listStaleChunks({ batchSize: 100 })).length).toBeGreaterThan(0);
|
||||
// updated_desc arm uses the same predicate.
|
||||
expect(await engine.listStaleChunks({ embeddingColumn: descriptor, orderBy: 'updated_desc', batchSize: 100 })).toHaveLength(0);
|
||||
}, 30_000);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -763,48 +763,3 @@ describeBoth('Engine parity — federated sourceIds[] secondary reads (#2200)',
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// #1144 — brain_score is composition-aware: code / calendar-index pages are
|
||||
// orphans-by-design and must not feed the orphan/link-density/timeline
|
||||
// denominators. Both engines must agree.
|
||||
async function seedComposition(eng: BrainEngine) {
|
||||
await eng.putPage('people/comp-alice', { type: 'person', title: 'Alice', compiled_truth: 'a', timeline: '' });
|
||||
await eng.putPage('people/comp-bob', { type: 'person', title: 'Bob', compiled_truth: 'b', timeline: '' });
|
||||
await eng.addLink('people/comp-alice', 'people/comp-bob', 'knows', 'mentions', 'markdown');
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await eng.putPage(`code/src/comp-f${i}.py`, { type: 'code', title: `f${i}.py`, compiled_truth: 'def f(): pass', timeline: '' });
|
||||
}
|
||||
await eng.putPage('daily/calendar/2026-01-01', { type: 'calendar-index', title: '2026-01-01', compiled_truth: 'events', timeline: '' });
|
||||
}
|
||||
|
||||
describeBoth('Engine parity — getHealth composition-aware brain_score (#1144)', () => {
|
||||
let pgEngine: BrainEngine;
|
||||
let pgliteEngine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
pgEngine = await setupDB();
|
||||
await seedComposition(pgEngine);
|
||||
pgliteEngine = new PGLiteEngine();
|
||||
await pgliteEngine.connect({});
|
||||
await pgliteEngine.initSchema();
|
||||
await seedComposition(pgliteEngine);
|
||||
}, 90_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await pgliteEngine.disconnect();
|
||||
await teardownDB();
|
||||
}, 30_000);
|
||||
|
||||
test('orphan/link-density/timeline components identical and exclude code + calendar-index pages', async () => {
|
||||
const pg = await pgEngine.getHealth();
|
||||
const pglite = await pgliteEngine.getHealth();
|
||||
for (const k of ['orphan_pages', 'no_orphans_score', 'link_density_score', 'timeline_coverage_score', 'brain_score'] as const) {
|
||||
expect(pg[k]).toBe(pglite[k]);
|
||||
}
|
||||
// 6 non-narrative pages exist but both narrative pages are linked → 0 orphans.
|
||||
expect(pg.orphan_pages).toBe(0);
|
||||
expect(pg.no_orphans_score).toBe(15);
|
||||
// 1 link / 2 narrative pages, not / 8 total pages.
|
||||
expect(pg.link_density_score).toBe(Math.round(0.5 * 25));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -172,59 +172,6 @@ describe('issue #972 — DB-source (gbrain extract links --source db)', () => {
|
||||
expect(strk!.link_type).toBe('wikilink_basename');
|
||||
});
|
||||
|
||||
test('flag ON → path-qualified wikilink outside DIR_PATTERN resolves via DB path', async () => {
|
||||
// `[[notes/struktura]]` — `notes` is not in DIR_PATTERN, so the ref
|
||||
// reaches the generic pass with its dirname intact. Regression: the DB
|
||||
// path queried the basename index with the raw literal (which is keyed
|
||||
// by final segments only), so path-qualified wikilinks outside
|
||||
// DIR_PATTERN silently produced zero edges while the FS path resolved
|
||||
// the identical content.
|
||||
await engine.putPage('notes/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Notes',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/knowledge-graph', {
|
||||
type: 'concept', title: 'Knowledge Graph',
|
||||
compiled_truth: 'Background in [[notes/struktura]].', timeline: '',
|
||||
});
|
||||
await engine.setConfig('link_resolution.global_basename', 'true');
|
||||
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
|
||||
const outLinks = await engine.getLinks('concepts/knowledge-graph');
|
||||
const strk = outLinks.find(l => l.to_slug === 'notes/struktura');
|
||||
expect(strk).toBeDefined();
|
||||
expect(strk!.link_type).toBe('wikilink_basename');
|
||||
expect(strk!.link_source).toBe('wikilink-resolved');
|
||||
});
|
||||
|
||||
test('path-qualified wikilink never attaches to a basename-only sibling', async () => {
|
||||
// Both notes/struktura and wiki/struktura exist. The author wrote
|
||||
// `[[notes/struktura]]` — the written path must exclude wiki/struktura
|
||||
// (a bare `[[struktura]]` would legitimately match both).
|
||||
await engine.putPage('notes/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Notes',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('wiki/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Wiki',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/x', {
|
||||
type: 'concept', title: 'X',
|
||||
compiled_truth: 'See [[notes/struktura]].', timeline: '',
|
||||
});
|
||||
await engine.setConfig('link_resolution.global_basename', 'true');
|
||||
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
|
||||
const outLinks = await engine.getLinks('concepts/x');
|
||||
const basenameLinks = outLinks
|
||||
.filter(l => l.link_type === 'wikilink_basename')
|
||||
.map(l => l.to_slug);
|
||||
expect(basenameLinks).toEqual(['notes/struktura']);
|
||||
});
|
||||
|
||||
test('flag OFF → no basename edges via DB path (back-compat)', async () => {
|
||||
await engine.putPage('projects/struktura', {
|
||||
type: 'project', title: 'Struktura',
|
||||
|
||||
@@ -140,35 +140,6 @@ describeE2E('scanIntegrity batch parity (E2E, Postgres-only)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('code pages', () => {
|
||||
test('type:code page is skipped on both paths (#1143)', async () => {
|
||||
const engine = getEngine();
|
||||
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person',
|
||||
title: 'Alice',
|
||||
compiled_truth: 'Alice tweeted about something.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
await engine.putPage('code/src/bot.py', {
|
||||
type: 'code',
|
||||
title: 'src/bot.py (python)',
|
||||
compiled_truth: '# the user tweeted about this feature\ndef send_tweet():\n pass',
|
||||
timeline: '',
|
||||
frontmatter: { language: 'python', file: 'src/bot.py' },
|
||||
});
|
||||
|
||||
const batchResult = await scanIntegrity(engine, { limit: 100, batchLoad: true });
|
||||
const seqResult = await scanIntegrity(engine, { limit: 100, batchLoad: false });
|
||||
|
||||
expect(batchResult.pagesScanned).toBe(seqResult.pagesScanned);
|
||||
expect(batchResult.pagesScanned).toBe(1);
|
||||
expect(batchResult.bareHits.map(h => h.slug)).not.toContain('code/src/bot.py');
|
||||
expect(seqResult.bareHits.map(h => h.slug)).not.toContain('code/src/bot.py');
|
||||
});
|
||||
});
|
||||
|
||||
describe('topPages', () => {
|
||||
test('topPages ordering matches between paths', async () => {
|
||||
const engine = getEngine();
|
||||
|
||||
@@ -39,7 +39,6 @@ import { runSkillOpt } from '../../src/core/skillopt/orchestrator.ts';
|
||||
import {
|
||||
bestPath,
|
||||
loadHistory,
|
||||
proposedPath,
|
||||
skillPath,
|
||||
} from '../../src/core/skillopt/version-store.ts';
|
||||
import { loadRejectedBuffer } from '../../src/core/skillopt/rejected-buffer.ts';
|
||||
@@ -742,7 +741,7 @@ describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', ()
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('--no-mutate writes proposed.md and best.md, leaves SKILL.md untouched', async () => {
|
||||
test('--no-mutate writes proposed.md (best.md), leaves SKILL.md untouched', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
@@ -754,9 +753,10 @@ describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', ()
|
||||
const result = await runOnce(fixture, { noMutate: true });
|
||||
expect(result.outcome).toBe('accepted');
|
||||
expect(result.mutatedSkillFile).toBe(false);
|
||||
expect(result.proposedPath).toBe(proposedPath(fixture.skillsDir, SKILL));
|
||||
expect(result.proposedPath).toBeDefined();
|
||||
// proposed.md (best.md) exists and carries the improvement.
|
||||
expect(fs.existsSync(result.proposedPath!)).toBe(true);
|
||||
expect(fs.readFileSync(result.proposedPath!, 'utf8')).toContain('## Citations');
|
||||
expect(fs.readFileSync(bestPath(fixture.skillsDir, SKILL), 'utf8')).toContain('## Citations');
|
||||
// SKILL.md on disk is UNCHANGED (still People-only).
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).not.toContain('## Citations');
|
||||
|
||||
@@ -803,107 +803,3 @@ describe('embedAllStale --source threading (D7)', () => {
|
||||
expect((firstCallOpts as { sourceId?: string }).sourceId).toBe('media-corpus');
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// Code metadata preservation across re-embed (regression for #769)
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// gbrain v0.30.1 and earlier silently clobbered code-chunk metadata
|
||||
// (language, symbol_name, symbol_type, start_line, end_line,
|
||||
// parent_symbol_path, doc_comment, symbol_name_qualified) on every
|
||||
// re-embed pass. The chunker populated those columns at import time,
|
||||
// but embed.ts loaded chunks via getChunks then mapped them to a
|
||||
// stripped ChunkInput carrying only 5 fields. upsertChunks then
|
||||
// OVERWROTE (not COALESCEd) the metadata columns from EXCLUDED, so
|
||||
// re-embed wiped them to NULL. End result on a real brain: 4875 code
|
||||
// pages, 47866 chunks, all with NULL language/symbol_name/symbol_type;
|
||||
// code-def returned 0 hits across every indexed repo.
|
||||
//
|
||||
// All three runEmbed paths (--stale autopilot, --all, --slugs) must
|
||||
// thread metadata through the re-upsert. Tests below assert that the
|
||||
// engine.upsertChunks call carries the same metadata it loaded.
|
||||
|
||||
describe('runEmbed preserves code-chunk metadata across re-embed (regression for #769)', () => {
|
||||
const fullCodeChunk = {
|
||||
chunk_index: 0,
|
||||
chunk_text: '[Java] foo/Bar.java:10-20 method baz',
|
||||
chunk_source: 'compiled_truth' as const,
|
||||
embedded_at: null,
|
||||
token_count: 12,
|
||||
language: 'java',
|
||||
symbol_name: 'baz',
|
||||
symbol_type: 'function',
|
||||
start_line: 10,
|
||||
end_line: 20,
|
||||
parent_symbol_path: ['Bar'],
|
||||
doc_comment: 'does the thing',
|
||||
symbol_name_qualified: 'Bar.baz',
|
||||
};
|
||||
|
||||
function metadataOf(chunk: any) {
|
||||
return {
|
||||
language: chunk.language,
|
||||
symbol_name: chunk.symbol_name,
|
||||
symbol_type: chunk.symbol_type,
|
||||
start_line: chunk.start_line,
|
||||
end_line: chunk.end_line,
|
||||
parent_symbol_path: chunk.parent_symbol_path,
|
||||
doc_comment: chunk.doc_comment,
|
||||
symbol_name_qualified: chunk.symbol_name_qualified,
|
||||
};
|
||||
}
|
||||
|
||||
test('--stale (autopilot path) carries code metadata into upsertChunks', async () => {
|
||||
const stale = [{
|
||||
slug: 'code-page',
|
||||
chunk_index: 0,
|
||||
chunk_text: fullCodeChunk.chunk_text,
|
||||
chunk_source: 'compiled_truth',
|
||||
model: null,
|
||||
token_count: 12,
|
||||
}];
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
countStaleChunks: async () => 1,
|
||||
listStaleChunks: async () => stale,
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--stale']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
|
||||
test('--all (full re-embed) carries code metadata into upsertChunks', async () => {
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
listPages: async () => [{ slug: 'code-page' }],
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--all']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
|
||||
test('--slugs (per-page embed) carries code metadata into upsertChunks', async () => {
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
getPage: async () => ({ slug: 'code-page', compiled_truth: 'x', timeline: '' }),
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--slugs', 'code-page']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -201,15 +201,6 @@ describe('scanIntegrity', () => {
|
||||
timeline: '',
|
||||
frontmatter: { validate: false },
|
||||
});
|
||||
// Indexed source file — 'tweeted about' in a comment must NOT be flagged
|
||||
// as a bare-tweet citation gap (#1143: auto-repair would corrupt source).
|
||||
await engine.putPage('code/src/bot.py', {
|
||||
type: 'code',
|
||||
title: 'src/bot.py (python)',
|
||||
compiled_truth: '# the user tweeted about this feature\ndef send_tweet():\n pass',
|
||||
timeline: '',
|
||||
frontmatter: { language: 'python', file: 'src/bot.py' },
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -232,13 +223,6 @@ describe('scanIntegrity', () => {
|
||||
expect(slugs).not.toContain('people/legacy');
|
||||
});
|
||||
|
||||
test('skips type:code pages (#1143 — bare-tweet false positives on source files)', async () => {
|
||||
const res = await scanIntegrity(engine);
|
||||
const slugs = res.bareHits.map(h => h.slug);
|
||||
expect(slugs).not.toContain('code/src/bot.py');
|
||||
expect(res.pagesScanned).toBe(2);
|
||||
});
|
||||
|
||||
test('honors limit', async () => {
|
||||
const res = await scanIntegrity(engine, { limit: 1 });
|
||||
expect(res.pagesScanned).toBe(1);
|
||||
|
||||
@@ -403,77 +403,6 @@ describe('extractPageLinks', () => {
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
|
||||
test('path-qualified wikilink outside DIR_PATTERN queries by final segment', async () => {
|
||||
// `[[notes/struktura]]` (dir not in DIR_PATTERN) falls to the generic
|
||||
// pass. The resolver's basename index is keyed by final path segments,
|
||||
// so the lookup must strip the dirname — mirroring the FS path
|
||||
// (resolveSlugAll). Regression: the raw literal was passed through,
|
||||
// which never matched, so these links silently dropped.
|
||||
const seen: string[] = [];
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) => {
|
||||
seen.push(name);
|
||||
return name === 'struktura' ? ['notes/struktura'] : [];
|
||||
},
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(seen).toContain('struktura');
|
||||
expect(seen).not.toContain('notes/struktura');
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura']);
|
||||
expect(candidates[0].linkType).toBe('wikilink_basename');
|
||||
expect(candidates[0].linkSource).toBe('wikilink-resolved');
|
||||
});
|
||||
|
||||
test('path-qualified wikilink keeps only matches ending with the written path', async () => {
|
||||
// The written path disambiguates: `[[notes/struktura]]` must never
|
||||
// attach to `wiki/struktura` even though both share the basename.
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['notes/struktura', 'wiki/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura']);
|
||||
});
|
||||
|
||||
test('path-qualified wikilink matches a deeper real slug by path suffix', async () => {
|
||||
// The page lives at vault/notes/struktura; the author wrote the shorter
|
||||
// tail `[[notes/struktura]]`. Suffix matching connects them, while the
|
||||
// basename-only sibling `wiki/struktura` stays excluded.
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['vault/notes/struktura', 'wiki/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['vault/notes/struktura']);
|
||||
});
|
||||
|
||||
test('path-qualified self-link is dropped like the bare form', async () => {
|
||||
// `[[notes/struktura]]` written on notes/struktura itself must not
|
||||
// produce a self-loop (same guard as the bare `[[own-tail]]` case).
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['notes/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'notes/struktura', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
|
||||
test('bare wikilink resolution does not interfere with DIR_PATTERN wikilinks', async () => {
|
||||
// 2b refs (people/alice) take the verb-inferred type;
|
||||
// 2c refs (struktura) take wikilink_basename. Same call.
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import {
|
||||
extractCycleFreshnessSourceIds,
|
||||
parseMaintainArgs,
|
||||
} from '../src/commands/maintain.ts';
|
||||
import type { Check } from '../src/commands/doctor.ts';
|
||||
|
||||
describe('maintain args', () => {
|
||||
test('defaults to dry-run unless --safe is explicit', () => {
|
||||
expect(parseMaintainArgs([])).toMatchObject({
|
||||
safe: false,
|
||||
dryRun: true,
|
||||
json: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('--safe enables mutating safe mode', () => {
|
||||
expect(parseMaintainArgs(['--safe', '--json'])).toMatchObject({
|
||||
safe: true,
|
||||
dryRun: false,
|
||||
json: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('--dry-run wins over --safe', () => {
|
||||
expect(parseMaintainArgs(['--safe', '--dry-run'])).toMatchObject({
|
||||
safe: true,
|
||||
dryRun: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('cycle freshness source extraction', () => {
|
||||
test('extracts stale source ids from doctor messages', () => {
|
||||
const checks: Check[] = [
|
||||
{
|
||||
name: 'cycle_freshness',
|
||||
status: 'fail',
|
||||
message: "Source 'brain-sync-remote-teffur' last cycled 40h ago. Run `gbrain dream --source <id>`.",
|
||||
},
|
||||
{
|
||||
name: 'cycle_freshness',
|
||||
status: 'fail',
|
||||
message: "Source 'wiki' last cycled 25h ago. Source 'wiki' last cycled 25h ago.",
|
||||
},
|
||||
];
|
||||
|
||||
expect(extractCycleFreshnessSourceIds(checks)).toEqual([
|
||||
'brain-sync-remote-teffur',
|
||||
'wiki',
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores ok and unrelated checks', () => {
|
||||
const checks: Check[] = [
|
||||
{ name: 'cycle_freshness', status: 'ok', message: "Source 'fresh' last cycled recently." },
|
||||
{ name: 'frontmatter_integrity', status: 'warn', message: "Source 'wiki' has frontmatter issues." },
|
||||
];
|
||||
|
||||
expect(extractCycleFreshnessSourceIds(checks)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,17 +0,0 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
describe('root OpenClaw plugin manifest', () => {
|
||||
it('declares the id required by OpenClaw plugin installs', () => {
|
||||
const manifest = JSON.parse(readFileSync(join(import.meta.dir, '..', 'openclaw.plugin.json'), 'utf8'));
|
||||
const entrySource = readFileSync(join(import.meta.dir, '..', 'src', 'openclaw-context-engine.ts'), 'utf8');
|
||||
const entryId = entrySource.match(/id:\s*'([^']+)'/)?.[1];
|
||||
|
||||
expect(manifest.id).toBe(entryId);
|
||||
expect(manifest.configSchema).toBeDefined();
|
||||
expect(typeof manifest.configSchema).toBe('object');
|
||||
expect(manifest.contracts?.contextEngines).toContain('gbrain-context');
|
||||
expect(entrySource).toContain('export function register');
|
||||
});
|
||||
});
|
||||
@@ -186,67 +186,11 @@ describe('shouldExclude — orphan filter regression (preserve curation)', () =>
|
||||
expect(shouldExclude('entities/anonymous')).toBe(true);
|
||||
expect(shouldExclude('atoms/fact-123')).toBe(true);
|
||||
expect(shouldExclude('skills/gbrain-operations')).toBe(true);
|
||||
expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('daily/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('workspace convention slugs are excluded', () => {
|
||||
expect(shouldExclude('_brain-conventions')).toBe(true);
|
||||
expect(shouldExclude('_templates/decision')).toBe(true);
|
||||
expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/identity')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('regular slugs are NOT excluded', () => {
|
||||
expect(shouldExclude('people/alice')).toBe(false);
|
||||
expect(shouldExclude('companies/acme')).toBe(false);
|
||||
expect(shouldExclude('writing/post-1')).toBe(false);
|
||||
expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHealth orphan_pages uses shared exclusion policy', () => {
|
||||
test('excluded convention islands do not count against health', async () => {
|
||||
await engine.putPage('_templates/decision', {
|
||||
type: 'template', title: 'Decision', compiled_truth: 'template', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('skills/arya/source-check', {
|
||||
type: 'concept', title: 'Skill', compiled_truth: 'skill', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('agents/arya/identity', {
|
||||
type: 'note', title: 'Identity', compiled_truth: 'identity', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {},
|
||||
});
|
||||
|
||||
const health = await engine.getHealth();
|
||||
|
||||
expect(health.orphan_pages).toBe(1);
|
||||
});
|
||||
|
||||
test('per-brain config overrides (orphans.exclude_*) also apply to health', async () => {
|
||||
await engine.putPage('my-private-folder/secret-ref', {
|
||||
type: 'note', title: 'Ref', compiled_truth: 'ref', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('one-off-fixture-page', {
|
||||
type: 'note', title: 'Fixture', compiled_truth: 'fixture', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {},
|
||||
});
|
||||
|
||||
expect((await engine.getHealth()).orphan_pages).toBe(3);
|
||||
|
||||
await engine.setConfig('orphans.exclude_prefixes', 'my-private-folder/');
|
||||
await engine.setConfig('orphans.exclude_slugs', 'one-off-fixture-page');
|
||||
expect((await engine.getHealth()).orphan_pages).toBe(1);
|
||||
|
||||
await engine.unsetConfig('orphans.exclude_prefixes');
|
||||
await engine.unsetConfig('orphans.exclude_slugs');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -66,10 +66,6 @@ describe('shouldExclude', () => {
|
||||
expect(shouldExclude('templates/meeting-note')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: _templates/', () => {
|
||||
expect(shouldExclude('_templates/meeting-note')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: openclaw/config/', () => {
|
||||
expect(shouldExclude('openclaw/config/agent')).toBe(true);
|
||||
});
|
||||
@@ -90,44 +86,10 @@ describe('shouldExclude', () => {
|
||||
expect(shouldExclude('entities/product-hunt')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: skills, dreaming, and daily', () => {
|
||||
expect(shouldExclude('skills/arya/source-check')).toBe(true);
|
||||
expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('daily/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes root date logs and agent workspace conventions', () => {
|
||||
expect(shouldExclude('_brain-conventions')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/identity')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes generated extracts', () => {
|
||||
expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true);
|
||||
});
|
||||
|
||||
test('brain-specific exclusions come from config overrides, not global defaults', () => {
|
||||
// No baked-in defaults for these:
|
||||
expect(shouldExclude('my-private-folder/some-secret-ref.md')).toBe(false);
|
||||
expect(shouldExclude('one-off-fixture-page')).toBe(false);
|
||||
// The per-brain config plane (orphans.exclude_prefixes / exclude_slugs):
|
||||
const overrides = {
|
||||
excludePrefixes: ['my-private-folder/'],
|
||||
excludeSlugs: ['one-off-fixture-page'],
|
||||
};
|
||||
expect(shouldExclude('my-private-folder/some-secret-ref.md', overrides)).toBe(true);
|
||||
expect(shouldExclude('one-off-fixture-page', overrides)).toBe(true);
|
||||
expect(shouldExclude('people/jane-doe', overrides)).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT exclude a normal content page', () => {
|
||||
expect(shouldExclude('companies/acme')).toBe(false);
|
||||
expect(shouldExclude('people/jane-doe')).toBe(false);
|
||||
expect(shouldExclude('projects/gbrain')).toBe(false);
|
||||
expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT exclude a page ending with log-like text that is not /log', () => {
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
/**
|
||||
* #1098 — ClawVisor health checks must hit /ready (surfaces db/vault
|
||||
* degradation), never /health (bare liveness, masks those failure modes).
|
||||
* #1099 — the credential-gateway recipe tells users to activate Google
|
||||
* Contacts; the canonical consumer recipe must actually ship.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const RECIPES = join(import.meta.dir, '../recipes');
|
||||
|
||||
describe('#1098 — ClawVisor health checks use /ready', () => {
|
||||
for (const f of ['email-to-brain.md', 'calendar-to-brain.md', 'credential-gateway.md', 'contacts-to-brain.md']) {
|
||||
test(`${f} references $CLAWVISOR_URL/ready, never /health`, () => {
|
||||
const text = readFileSync(join(RECIPES, f), 'utf-8');
|
||||
expect(text).not.toContain('$CLAWVISOR_URL/health');
|
||||
expect(text).toContain('$CLAWVISOR_URL/ready');
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('#1099 — canonical contacts-to-brain recipe ships', () => {
|
||||
test('recipes/contacts-to-brain.md exists with the sibling frontmatter shape', () => {
|
||||
const p = join(RECIPES, 'contacts-to-brain.md');
|
||||
expect(existsSync(p)).toBe(true);
|
||||
const text = readFileSync(p, 'utf-8');
|
||||
expect(text).toContain('id: contacts-to-brain');
|
||||
expect(text).toContain('requires: [credential-gateway]');
|
||||
expect(text).toContain('health_checks:');
|
||||
expect(text).toContain('contacts.readonly');
|
||||
});
|
||||
});
|
||||
@@ -13,9 +13,10 @@
|
||||
* throw on unknown string.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { describe, test, expect, afterAll, afterEach } from 'bun:test';
|
||||
import {
|
||||
resolveEmbeddingColumn,
|
||||
resolveWriteColumn,
|
||||
getEmbeddingColumnRegistry,
|
||||
buildVectorCastFragment,
|
||||
quoteIdentifier,
|
||||
@@ -34,6 +35,28 @@ import {
|
||||
} from '../../src/core/search/embedding-column.ts';
|
||||
import type { GBrainConfig } from '../../src/core/config.ts';
|
||||
import type { ResolvedColumn } from '../../src/core/types.ts';
|
||||
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
|
||||
|
||||
/**
|
||||
* Teardown: reset AND re-apply the legacy preload config
|
||||
* (test/helpers/legacy-embedding-preload.ts). A bare resetGateway() would
|
||||
* leave the slot empty for the NEXT file's beforeAll (the preload's
|
||||
* per-test beforeEach only fires before tests, not before beforeAll), which
|
||||
* would make sibling PGLite fixtures initSchema at the 1280 default instead
|
||||
* of the legacy 1536 their seed vectors assume.
|
||||
*/
|
||||
function restorePreloadGateway() {
|
||||
resetGateway();
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
|
||||
afterAll(() => {
|
||||
restorePreloadGateway();
|
||||
});
|
||||
|
||||
function cfg(overrides: Partial<GBrainConfig> = {}): GBrainConfig {
|
||||
return { engine: 'pglite', ...overrides };
|
||||
@@ -522,3 +545,89 @@ describe('codex /ship #4 — isCacheSafe (embedding-space-based skip)', () => {
|
||||
expect(isCacheSafe(r, cfg())).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveWriteColumn — write-side boundary resolution (#1262)', () => {
|
||||
afterEach(() => {
|
||||
restorePreloadGateway();
|
||||
});
|
||||
|
||||
test('no registry / empty registry returns undefined (legacy single-column brain)', () => {
|
||||
expect(resolveWriteColumn(cfg())).toBeUndefined();
|
||||
expect(resolveWriteColumn(cfg({ embedding_columns: {} }))).toBeUndefined();
|
||||
});
|
||||
|
||||
test('provider match via cfg.embedding_model returns the descriptor', () => {
|
||||
const r = resolveWriteColumn(cfg({
|
||||
embedding_model: 'voyage:voyage-3-large',
|
||||
embedding_dimensions: 1024,
|
||||
embedding_columns: {
|
||||
embedding_voyage: { provider: 'voyage:voyage-3-large', dimensions: 1024, type: 'vector' },
|
||||
},
|
||||
}));
|
||||
expect(r).toEqual({
|
||||
name: 'embedding_voyage',
|
||||
type: 'vector',
|
||||
dimensions: 1024,
|
||||
embeddingModel: 'voyage:voyage-3-large',
|
||||
});
|
||||
});
|
||||
|
||||
test('provider match via gateway state (cfg.embedding_model unset) returns descriptor', () => {
|
||||
configureGateway({
|
||||
embedding_model: 'zeroentropyai:zembed-1',
|
||||
embedding_dimensions: 2560,
|
||||
env: {},
|
||||
});
|
||||
const r = resolveWriteColumn(cfg({
|
||||
embedding_columns: {
|
||||
embedding_ze: { provider: 'zeroentropyai:zembed-1', dimensions: 2560, type: 'halfvec' },
|
||||
},
|
||||
}));
|
||||
expect(r).toEqual({
|
||||
name: 'embedding_ze',
|
||||
type: 'halfvec',
|
||||
dimensions: 2560,
|
||||
embeddingModel: 'zeroentropyai:zembed-1',
|
||||
});
|
||||
});
|
||||
|
||||
test('no provider match returns undefined instead of guessing a column', () => {
|
||||
configureGateway({
|
||||
embedding_model: 'zeroentropyai:zembed-1',
|
||||
embedding_dimensions: 2560,
|
||||
env: {},
|
||||
});
|
||||
const r = resolveWriteColumn(cfg({
|
||||
embedding_columns: {
|
||||
embedding_voyage: { provider: 'voyage:voyage-3-large', dimensions: 1024, type: 'vector' },
|
||||
},
|
||||
}));
|
||||
expect(r).toBeUndefined();
|
||||
});
|
||||
|
||||
test('only USER-declared columns are consulted — multimodal builtin never captures text writes', () => {
|
||||
// Current model equals the embedding_image BUILTIN's provider; a registry
|
||||
// walk that consulted builtins would misroute text writes into the image
|
||||
// column. resolveWriteColumn must return undefined here.
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-multimodal-3',
|
||||
embedding_dimensions: 1024,
|
||||
env: {},
|
||||
});
|
||||
const r = resolveWriteColumn(cfg({
|
||||
embedding_columns: {
|
||||
embedding_other: { provider: 'openai:text-embedding-3-large', dimensions: 1536, type: 'vector' },
|
||||
},
|
||||
}));
|
||||
expect(r).toBeUndefined();
|
||||
});
|
||||
|
||||
test('malformed registry entry throws loud (same validation as the read side)', () => {
|
||||
expect(() => resolveWriteColumn(cfg({
|
||||
embedding_model: 'voyage:voyage-3-large',
|
||||
embedding_columns: {
|
||||
'bad"col': { provider: 'voyage:voyage-3-large', dimensions: 1024, type: 'vector' },
|
||||
} as never,
|
||||
}))).toThrow(EmbeddingColumnConfigError);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,11 +12,9 @@ import {
|
||||
bestPath,
|
||||
historyPath,
|
||||
loadHistory,
|
||||
proposedPath,
|
||||
revertAllPending,
|
||||
skillPath,
|
||||
versionsDir,
|
||||
writeProposed,
|
||||
} from '../../src/core/skillopt/version-store.ts';
|
||||
|
||||
let tmpDir: string;
|
||||
@@ -81,19 +79,6 @@ describe('acceptCandidate (D8 two-phase commit)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('writeProposed', () => {
|
||||
test('writes distinct best and proposed artifacts without mutating SKILL.md (#2635)', () => {
|
||||
const candidate = '---\nname: test\n---\nproposed body\n';
|
||||
|
||||
const written = writeProposed(tmpDir, SKILL, candidate);
|
||||
|
||||
expect(written).toBe(proposedPath(tmpDir, SKILL));
|
||||
expect(fs.readFileSync(bestPath(tmpDir, SKILL), 'utf8')).toBe(candidate);
|
||||
expect(fs.readFileSync(proposedPath(tmpDir, SKILL), 'utf8')).toBe(candidate);
|
||||
expect(fs.readFileSync(skillPath(tmpDir, SKILL), 'utf8')).toContain('baseline body');
|
||||
});
|
||||
});
|
||||
|
||||
describe('revertAllPending (D8 crash recovery)', () => {
|
||||
test('no-op when no pending rows', () => {
|
||||
const reverted = revertAllPending(tmpDir, SKILL);
|
||||
|
||||
Reference in New Issue
Block a user