mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
Merge remote-tracking branch 'origin/master' into garrytan/cathedral-1
Catch up to master: 526 commits (up to v0.42.73.2 — CI PR-usefulness gate, OAuth slug-prefix write fence, GitHub-release publishing, community fix waves, dep CVE-pin overrides, Windows path fixes, silent-failure doctor batch, and more). Conflicts resolved (9): - VERSION / package.json: kept 0.43.0.0 (Cathedral 1 — higher semver); merged in master's new CVE-pin `overrides` block. - CHANGELOG.md / TODOS.md: kept both sides (ours on top, master's below). - src/cli.ts: CLI_ONLY = master's superset (now `export`) + our `protocol`; CLI_ONLY_SELF_HELP keeps both (`protocol` + master's `init`/`migrate`/`retrieval-upgrade`). - src/commands/serve-http.ts: ServeHttpOptions keeps both new fields (`surface` ours, `printAdminToken` master). - src/commands/serve.ts: runServeHttp call keeps both options + master's new finishHttpServe teardown; stdio path keeps master's boot-readiness deadline (#3273) while preserving our `start(engine, { surface })`. - src/core/config.ts: config allowlist keeps both (`mcp_surface`/`protocol_installed_at` + `provider_chat_options`). - src/core/think/index.ts: both sides added the same token-`usage` feature; unified to one nullable `usage` (interface field, local var, return) and kept master's additive `cost_usd?`. (Duplicate-identifier build break caught by typecheck and fixed.) Verified: 3-line version audit agrees on 0.43.0.0; bun install reconciled bun.lock against master's overrides; bun run typecheck clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -0,0 +1,29 @@
|
||||
# Line-ending policy.
|
||||
#
|
||||
# Shell scripts MUST be checked out with LF endings on every platform.
|
||||
# Git for Windows installs with `core.autocrlf=true` by default, which
|
||||
# rewrites LF -> CRLF on checkout. A strict bash (WSL, Linux CI, macOS)
|
||||
# then chokes on the trailing CR:
|
||||
#
|
||||
# scripts/run-unit-parallel.sh: line 23: $'\r': command not found
|
||||
# scripts/run-unit-parallel.sh: line 24: set: pipefail : invalid option name
|
||||
# scripts/run-unit-parallel.sh: line 32: syntax error near unexpected token `$'{\r''
|
||||
#
|
||||
# That silently disabled `bun run test`, `bun run verify`, `bun run ci:local`
|
||||
# and `bun run test:e2e` for Windows contributors, since all four dispatch
|
||||
# through bash. `eol=lf` pins the checkout regardless of the user's
|
||||
# core.autocrlf setting.
|
||||
*.sh text eol=lf
|
||||
|
||||
# Markdown gets the same pin, for a different failure mode: the frontmatter
|
||||
# parsers anchor on LF. Under a CRLF checkout the opening fence becomes
|
||||
# "---\r\n", which an LF-only /^---\n/ (or a startsWith("---\n")) does not
|
||||
# match, so a well-formed document silently parses as having no frontmatter.
|
||||
# There is no error -- the field just comes back empty. That has surfaced as
|
||||
# blank skill descriptions, a fixer inserting its banner above the
|
||||
# frontmatter instead of below it, resolver trigger extraction dropping
|
||||
# entries, and a generated-doc freshness check reporting every line as
|
||||
# drifted. The parsers stay CR-tolerant on their own merits (gbrain reads
|
||||
# Markdown it does not own), but pinning this repo's own .md checkout to LF
|
||||
# removes the whole class for anyone working here.
|
||||
*.md text eol=lf
|
||||
@@ -28,5 +28,5 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -61,7 +61,10 @@ jobs:
|
||||
- name: Run JSONB double-encode parity tests on real Postgres
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
run: bun test test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
# --timeout also raises bun's 5s default hook budget (beforeAll/afterAll
|
||||
# do NOT inherit a test's third-arg timeout; verified on bun 1.3.x).
|
||||
# Every runner script in scripts/ passes it; bare invocations must too.
|
||||
run: bun test --timeout=60000 test/e2e/op-checkpoint-jsonb-parity.test.ts test/e2e/jsonb-roundtrip.test.ts
|
||||
|
||||
tier1:
|
||||
name: Tier 1 (Mechanical)
|
||||
@@ -82,13 +85,13 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- name: Run Tier 1 E2E tests
|
||||
run: bun test test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
run: bun test --timeout=60000 test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
|
||||
@@ -116,7 +119,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -155,7 +158,7 @@ jobs:
|
||||
}
|
||||
EOF
|
||||
- name: Run Tier 2 skill tests
|
||||
run: bun test test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
run: bun test --timeout=60000 test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
name: OSV-Scanner
|
||||
|
||||
# Dependency vulnerability scan (#2182) via Google's official reusable
|
||||
# workflow. Runs weekly and on any PR that touches the dependency manifests.
|
||||
# Tokenless: needs zero secrets. Findings are reported in the job log and as
|
||||
# a SARIF artifact on the run; code-scanning upload is deliberately disabled
|
||||
# so the workflow stays read-only (no security-events: write).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'bun.lock'
|
||||
- 'package.json'
|
||||
schedule:
|
||||
- cron: '30 6 * * 1' # weekly, Monday 06:30 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
osv-scan:
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
# Required by the reusable workflow's own top-level permissions block —
|
||||
# GitHub validates the caller grants a superset AT STARTUP, even with
|
||||
# upload-sarif: false (nothing is actually uploaded; see #2117 upstream).
|
||||
security-events: write
|
||||
uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
|
||||
with:
|
||||
upload-sarif: false
|
||||
@@ -1,14 +1,67 @@
|
||||
name: Release
|
||||
|
||||
# Publishes a GitHub release for every VERSION bump that lands on master:
|
||||
# tag + title `v<VERSION>`, notes from that version's CHANGELOG.md entry,
|
||||
# compiled binaries attached (#3521).
|
||||
#
|
||||
# Why every bump: `gbrain check-update` resolves the latest version from the
|
||||
# VERSION file on master, but binary self-update
|
||||
# (src/core/binary-self-update.ts) downloads assets from `releases/latest`.
|
||||
# If releases lag VERSION, binary installs are told an upgrade exists that
|
||||
# self-update cannot apply. Keeping releases/latest == VERSION closes that gap.
|
||||
#
|
||||
# Idempotent: the `version` job skips build+release when a release for
|
||||
# v<VERSION> already exists WITH all expected assets. A half-published release
|
||||
# (tag exists / assets incomplete) is repaired on the next run — softprops
|
||||
# updates the existing release in place. Historical 3-segment tags are never
|
||||
# touched; a new 4-segment VERSION always mints a new tag.
|
||||
#
|
||||
# The asset names are a contract with expectedAssetName() in
|
||||
# src/core/binary-self-update.ts, pinned by test/release-workflow.test.ts.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
branches: [master]
|
||||
paths: [VERSION]
|
||||
workflow_dispatch: {} # manual first run / backfill of the current VERSION
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
version:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
exists: ${{ steps.v.outputs.exists }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- id: v
|
||||
name: Read VERSION and check for an existing complete release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
version="$(tr -d '[:space:]' < VERSION)"
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
# Complete = release exists AND carries every asset the self-updater
|
||||
# can request. A partial release must NOT short-circuit, so a re-run
|
||||
# can repair it.
|
||||
assets="$(gh release view "v$version" --repo "$GITHUB_REPOSITORY" \
|
||||
--json assets --jq '[.assets[].name] | sort | join(",")' 2>/dev/null || true)"
|
||||
if [ "$assets" = "gbrain-darwin-arm64,gbrain-linux-x64" ]; then
|
||||
echo "exists=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Release v$version already published with all assets — nothing to do."
|
||||
else
|
||||
echo "exists=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
build:
|
||||
needs: version
|
||||
if: needs.version.outputs.exists == 'false'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -19,30 +72,68 @@ jobs:
|
||||
target: bun-linux-x64
|
||||
artifact: gbrain-linux-x64
|
||||
runs-on: ${{ matrix.os }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # for attest-build-provenance (Sigstore OIDC)
|
||||
attestations: write # for attest-build-provenance
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- run: bun test
|
||||
# No test re-run here: the Test workflow already gated this exact SHA at
|
||||
# merge (10 shards + E2E). Re-running the whole suite serially on the
|
||||
# release runner is a flakier duplicate gate — it blocked the first
|
||||
# release on ambient-env tests (run 30698650484). The build job's gate
|
||||
# is the artifact itself: compile, then smoke-test the binary.
|
||||
- run: bun build --compile --target=${{ matrix.target }} --outfile bin/${{ matrix.artifact }} src/cli.ts
|
||||
- name: Smoke-test the compiled binary
|
||||
run: |
|
||||
chmod +x bin/${{ matrix.artifact }}
|
||||
out="$(./bin/${{ matrix.artifact }} --version)"
|
||||
echo "binary reports: $out"
|
||||
v="$(tr -d '[:space:]' < VERSION)"
|
||||
case "$out" in *"$v"*) echo "version matches VERSION file" ;; *) echo "binary version '$out' does not contain '$v'" >&2; exit 1 ;; esac
|
||||
- name: Attest build provenance
|
||||
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
|
||||
with:
|
||||
subject-path: bin/${{ matrix.artifact }}
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ${{ matrix.artifact }}
|
||||
path: bin/${{ matrix.artifact }}
|
||||
|
||||
release:
|
||||
needs: build
|
||||
needs: [version, build]
|
||||
if: needs.version.outputs.exists == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # create the tag + release (scoped to this job only)
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Extract CHANGELOG entry for release notes
|
||||
# env-bound, not inlined into the script: VERSION comes from master so
|
||||
# it isn't attacker-reachable today, but a `${{ }}` inside `run:` is
|
||||
# shell injection by construction if that ever changes.
|
||||
env:
|
||||
RELEASE_VERSION: ${{ needs.version.outputs.version }}
|
||||
run: |
|
||||
v="$RELEASE_VERSION"
|
||||
if ! bash scripts/changelog-entry.sh "$v" > /tmp/release-notes.md || ! [ -s /tmp/release-notes.md ]; then
|
||||
echo "See [CHANGELOG.md](https://github.com/${GITHUB_REPOSITORY}/blob/master/CHANGELOG.md) for v$v." > /tmp/release-notes.md
|
||||
fi
|
||||
- name: Create release
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
tag_name: v${{ needs.version.outputs.version }}
|
||||
name: v${{ needs.version.outputs.version }}
|
||||
target_commitish: ${{ github.sha }}
|
||||
body_path: /tmp/release-notes.md
|
||||
fail_on_unmatched_files: true
|
||||
files: |
|
||||
artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64
|
||||
artifacts/gbrain-linux-x64/gbrain-linux-x64
|
||||
generate_release_notes: true
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
name: Semgrep
|
||||
|
||||
# Static analysis (SAST) with Semgrep Community Edition (#2272). Tokenless:
|
||||
# uses the public registry rulesets, needs zero secrets. Findings print in
|
||||
# the job log; no code-scanning/SARIF upload by design (keeps permissions
|
||||
# read-only, no security-events: write).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
schedule:
|
||||
- cron: '30 7 * * 1' # weekly, Monday 07:30 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
semgrep:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
container:
|
||||
image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
# Non-blocking initially (continue-on-error): the first runs establish a
|
||||
# baseline without failing unrelated PRs. Graduation path: once the
|
||||
# baseline findings are triaged (fixed or `# nosemgrep`'d), remove
|
||||
# continue-on-error so new findings block PRs.
|
||||
- name: Semgrep scan (report-only)
|
||||
run: semgrep scan --config p/default --config p/typescript --error
|
||||
continue-on-error: true
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||||
hash: ${{ steps.compute.outputs.hash }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- name: Compute content hash
|
||||
id: compute
|
||||
run: |
|
||||
@@ -84,10 +84,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
||||
- uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -113,6 +113,11 @@ jobs:
|
||||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||||
- run: bun install
|
||||
- run: bun run verify
|
||||
# Guard: no bare `bun test` in workflows/scripts — bun ignores
|
||||
# bunfig.toml's timeout, and hooks (beforeAll/afterAll) get the 5s
|
||||
# default regardless of per-test third-arg timeouts. Runs directly
|
||||
# (not via verify's CHECKS array) to avoid a package.json edit.
|
||||
- run: bash scripts/check-bun-test-timeout.sh
|
||||
|
||||
serial-tests:
|
||||
# *.serial.test.ts at --max-concurrency=1. Lives in its own runner so
|
||||
@@ -124,7 +129,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -149,7 +154,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -172,7 +177,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -219,13 +224,17 @@ jobs:
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
# 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a
|
||||
# shard past 15 min while every test is still passing — the timeout then
|
||||
# cancels the job and the test-status gate reads it as a failure. 13 runs
|
||||
# died this way on 2026-07-21/22 alone.
|
||||
timeout-minutes: 22
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
+10
-2
@@ -1,4 +1,7 @@
|
||||
node_modules/
|
||||
# No trailing slash: a bare `node_modules/` pattern matches directories only,
|
||||
# so a *symlink* named node_modules slips past it and can be committed
|
||||
# (that's how the /tmp-pointing symlink in faf5cdba got in). Match any type.
|
||||
node_modules
|
||||
bin/
|
||||
.DS_Store
|
||||
*.log
|
||||
@@ -15,7 +18,7 @@ supabase/.temp/
|
||||
# self-contained binaries (the bun --compile path embeds it via
|
||||
# `import path from 'admin/dist/index.html' with { type: 'file' }`).
|
||||
# Build via: cd admin && bun install && bun run build.
|
||||
admin/node_modules/
|
||||
admin/node_modules
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
@@ -35,6 +38,11 @@ export/
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Local agent instruction overrides (CLAUDE.local.md / AGENTS.local.md) — personal,
|
||||
# per-clone, loaded after the committed CLAUDE.md/AGENTS.md. Never committed.
|
||||
CLAUDE.local.md
|
||||
AGENTS.local.md
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
+705
@@ -49,6 +49,711 @@ complete operation catalog — both speak the verbs). Run `gbrain protocol confo
|
||||
to self-certify, and `gbrain protocol stats` to watch adoption. Memories your agent
|
||||
saves are readable by every agent connected to the brain by default; pass
|
||||
`visibility: "private"` for local-only facts.
|
||||
## [0.42.73.2] - 2026-08-05
|
||||
|
||||
**A write that deduplication redirects onto an existing page is now checked against the write scope of whoever asked for it.** When the same content arrives under a new slug, gbrain recognises it and points the write at the page that already holds it. That redirected target is now tested against the caller's own scope — under whichever mechanism confines that caller. One of the two mechanisms was consulted at that point; both are now.
|
||||
|
||||
Nothing changes for local CLI use, or for clients that hold unrestricted write access — neither was ever scope-confined. A confined caller whose write dedups onto a page **inside** its own scope keeps working exactly as before; that redirect is a feature and it is preserved, with a regression test to keep it that way. A confined caller whose write dedups onto a page **outside** its scope now gets `permission_denied`, with the remedy in the message: drop the `id:` frontmatter field, or change the content, to write a new page under your own prefix. The denial does not name the page the write resolved to.
|
||||
|
||||
Recommended for any brain served over HTTP to scope-restricted clients.
|
||||
|
||||
### To take advantage of v0.42.73.2
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
|
||||
Nothing to configure. Existing clients keep their scopes unchanged, and no re-registration is needed.
|
||||
|
||||
### For contributors
|
||||
|
||||
Reported privately by an external security researcher, who supplied a fix and a regression test with it. The version that shipped composes the two existing scope-matching rules into a single predicate rather than restating either one, so the check at the door and the check after a redirect cannot drift apart; the audit the report prompted closed the same gap on one further caller path.
|
||||
|
||||
## [0.42.73.1] - 2026-08-05
|
||||
|
||||
**Removes the PR gate that v0.42.73.0 added, and reverts the v0.42.72.1 contribution-policy change it enforced.** The gate cannot function on this repository, and it caused a real incident before that was understood.
|
||||
|
||||
The gate needed two things this repository does not grant it: an `ANTHROPIC_API_KEY` Actions secret for its verdict, and read-write workflow permissions to post a comment or set a label. Without them it can only skip. Worse, on its first live runs a read-only token turned every API call into a 403, the code treated that as a crash, and the check went red on an outside contributor's pull request four times with no comment explaining why. That was fixed in v0.42.73.0, but a check that runs on every pull request and can never reach a verdict does not earn its place in the repository.
|
||||
|
||||
The v0.42.72.1 contribution policy is also withdrawn: the human-written intent paragraph and gbrain-in-use screenshot are no longer required on issues and pull requests. `CONTRIBUTING.md`, both issue templates, and the pull-request template return to their pre-2026-08-02 state, and issues and PRs are reviewed on their content by maintainers, as before.
|
||||
|
||||
The code is preserved in git history at v0.42.73.0 and can be restored if the repository ever grants those permissions. If it is restored, the mechanical half — the intent and screenshot check, the version-first title rule, the red flags — should render to the Actions job summary instead of a comment, because that needs no token permission and no API key.
|
||||
|
||||
### To take advantage of v0.42.73.1
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
|
||||
Nothing to change. Everything else v0.42.73.0 shipped — the five contributed correctness fixes, `slug_filter`, and the four dependency pins that cleared six CVEs — is unaffected and stays.
|
||||
|
||||
## [0.42.73.0] - 2026-08-04
|
||||
|
||||
**Every incoming pull request now gets a verdict before anyone reads it — and five contributed fixes for silent wrong answers.**
|
||||
|
||||
**The PR gate.** Open a pull request against gbrain and an automated check now posts a single verdict comment within a minute: **merge-lane**, **close-lane**, or **needs-maintainer**, with its reasons and a checklist of what a human reviewer should verify for that specific diff. It also checks mechanically that the description carries the human-written intent paragraph and the screenshot of gbrain in use that `CONTRIBUTING.md` requires, and that the title leads with its version.
|
||||
|
||||
It is deliberately **advisory** — a triage signal and a reviewer checklist, not an authorization boundary. A green verdict is not permission to merge; a maintainer still decides. Pull-request code is never checked out or executed: the verdict comes from the description and the diff read through the API. Maintainer, bot, and draft pull requests are exempt from the intent-and-screenshot floor only (release automation cannot screenshot itself); they still receive the full verdict. Where the rubric can be argued with, the decision is taken away from it: a merge-lane recommendation is downgraded automatically when a diff adds a dependency, a new provider recipe, or new config keys, edits workflows, deletes a test, exceeds 40 files or 400 net source lines, or changes `src/` without touching a single test.
|
||||
|
||||
**Your import output parses again.** `gbrain import <dir> --json` printed five informational lines to stdout ahead of the JSON payload, so anything parsing that output read zero imports while its own bookkeeping recorded the files as ingested — and the next run skipped them permanently. Those lines now go to stderr under `--json`; human output is byte-for-byte unchanged.
|
||||
|
||||
**`sources harden --dry-run` no longer changes anything.** It reset the helper's executable bit before reaching the dry-run check, so a documented preview quietly mutated permissions.
|
||||
|
||||
**Telemetry records the model that actually ran.** Two nightly-cycle phases wrote a hardcoded or unrelated model name into their verdict cache, evidence signature, and spend metering while the gateway ran whatever chat model you configured. On any brain with a non-default model, the recorded history was fiction.
|
||||
|
||||
**`gbrain integrity` stops contradicting itself.** Dead-link findings were counted in the "Review queue" total but written to a different file, so `integrity review` disagreed with `integrity auto`'s own summary. They now get their own line.
|
||||
|
||||
**Retype rules can address API-ingested pages.** Mapping rules could only filter on a file path, which is empty for every page written through `put_page` — so no rule could target that whole class. A new `slug_filter` filters on the slug instead, and combines with the path filter when both are given.
|
||||
|
||||
Also: the `integrity` source comment no longer documents a `--dry-run` subcommand form that exits with an error.
|
||||
|
||||
### To take advantage of v0.42.73.0
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain import <dir> --json | jq . # now parses
|
||||
gbrain integrity auto # dead links reported separately
|
||||
```
|
||||
|
||||
Nothing to configure for the gate — it runs on pull requests to this repository. If you maintain a fork and want it, the workflow needs an `ANTHROPIC_API_KEY` secret; without one it skips loudly rather than blocking anyone.
|
||||
|
||||
### For contributors
|
||||
|
||||
The gate went through six rounds against two independent blind reviewers, each judging cold. The findings that changed the design most were not exploits but false positives: a code fence that swallowed the rest of a description, an explanation written as bullet points scoring zero words, a word floor stricter than the published policy, and a comment telling contributors to reopen a pull request that was never closed. Those four descriptions are now permanent regression fixtures — a gate that insults a first-time contributor is worse than no gate. Two properties are deliberate and documented rather than fixed: the mechanical floor is a floor (a determined author clears it in seconds), and a bare URL in a cited reason still autolinks.
|
||||
|
||||
Contributed by @YiconZiwei (#2655), @time-attack (#3764, #3759, #3726, #3751, #3739, and the gate groundwork in #3573/#3698).
|
||||
|
||||
## [0.42.72.1] - 2026-08-02
|
||||
|
||||
**Every issue and pull request now needs a human-written paragraph and a screenshot of gbrain actually being used.**
|
||||
|
||||
Effective immediately, opening an issue or a PR requires two things from you personally: a paragraph you wrote yourself saying why you're opening it — what you were doing, what went wrong or what you needed, why it matters — and a screenshot of your terminal, agent session, or logs showing the real situation. Rough grammar is fine and preferred over polish. AI-generated or AI-polished intent text is not accepted; the paragraph is the human part. AI assistance for the *code* is still welcome.
|
||||
|
||||
Issues and PRs missing either are closed without review, and can be reopened once both are added. Scrub private names, companies, keys, and brain contents from screenshots before attaching — a redacted screenshot is fine, a missing one is not.
|
||||
|
||||
The requirement is stated in `CONTRIBUTING.md` and pre-filled in the bug-report and feature-request issue templates plus a new pull-request template, so the fields are in front of you when you open one.
|
||||
|
||||
## [0.42.72.0] - 2026-08-01
|
||||
|
||||
**Per-person write isolation inside a shared source, and a guide for putting gbrain behind a multi-user agent harness.**
|
||||
|
||||
Until now, `--source` was the only write boundary: a client could write anywhere inside the source it was scoped to, and keeping each person in their own folder was a convention the agent had to honor by itself. Registering a client with `--bound-slug-prefixes` now makes that boundary real. Writes outside the bound prefixes are refused by the server, on every op that can name a page.
|
||||
|
||||
**Adding a binding to an existing client narrows it on purpose.** Ops that write by something other than a page slug can't be confined to a prefix, so a bound client is refused them outright rather than left with an unfenced path: `extract_entities`, `extract_facts`, `forget_fact`, `ontology_propose`, `sources_add`/`sources_remove`, and `POST /ingest`. `put_page`'s automatic fact extraction is skipped for the same reason — it writes to entity pages the caller never named. Reads are unaffected, and unbound clients behave exactly as before. The gate keys on "anything that is not a plain read", so an op added in a future release is refused to bound clients until it is explicitly fenced.
|
||||
|
||||
Both prefix spellings work: the `wiki/agents/alice/*` glob that `submit_agent` bindings already use, and the plainer `emp-alice/` form. Change a binding in place with `gbrain auth rescope-client <id> --bound-slug-prefixes <p1,p2|none>` — existing tokens pick it up on their next request, so no secret rotation is needed when someone joins or leaves a team.
|
||||
|
||||
**New guide: [gbrain as the company brain for a qm deployment](docs/integrations/qm-harness.md).** qm is a multiplayer agent harness where each employee and each channel gets an isolated agent scope. The guide covers the whole path — one central `gbrain serve --http`, the thin-client binary baked into the sandbox image, one OAuth client per scope, and a roster-driven provisioning script that converges the brain to a list of people and channels. It also states plainly what the model does *not* give you: within a shared source, reads stay source-granular, so prefix isolation is a write boundary, not a privacy boundary.
|
||||
gbrain upgrade # or: bun install -g gbrain@0.42.72.0
|
||||
gbrain apply-migrations --yes # required: the fence refuses writes it cannot evaluate
|
||||
```
|
||||
|
||||
To fence an existing client to a folder:
|
||||
|
||||
```bash
|
||||
gbrain auth rescope-client <client_id> --bound-slug-prefixes partners/alice-example/
|
||||
gbrain auth rescope-client <client_id> --bound-slug-prefixes none # undo
|
||||
```
|
||||
|
||||
Verify it took, from a client holding that credential — the first write should succeed and the second should be refused:
|
||||
|
||||
```bash
|
||||
gbrain put partners/alice-example/notes/test --content "mine"
|
||||
gbrain put partners/bob-example/notes/test --content "not mine"
|
||||
```
|
||||
|
||||
## [0.42.71.0] - 2026-08-01
|
||||
|
||||
**GBrain now publishes real releases. Every version bump from here on lands on the [Releases page](https://github.com/garrytan/gbrain/releases) with organized notes and downloadable binaries — and binary self-update finally works.**
|
||||
|
||||
Until now the repo had no releases at all: `gbrain check-update` could tell you a new version existed, but `gbrain self-upgrade` downloaded from an empty releases API and failed every time, and anyone trying to follow what shipped had to read raw commit history. That's what people have been (rightly) complaining about.
|
||||
|
||||
From this release forward, every version bump automatically:
|
||||
|
||||
- **Tags the commit** (`v0.42.71.0`) so versions are addressable in git.
|
||||
- **Publishes a GitHub Release** whose notes are that version's CHANGELOG entry — the same organized, user-facing writeup, not a commit dump.
|
||||
- **Attaches compiled binaries** for macOS (arm64) and Linux (x64), so `gbrain self-upgrade` and fresh binary installs work without a toolchain.
|
||||
|
||||
The pipeline is idempotent: a partial release (tag exists, assets incomplete) is repaired on the next run instead of wedging. It runs post-merge, so a flaky release build can never turn master red. Releases for today's two fix waves (v0.42.69.0 and v0.42.70.0) have been backfilled with their CHANGELOG notes so the Releases page tells the whole story of the day; binaries attach from v0.42.71.0 onward.
|
||||
|
||||
### To take advantage of v0.42.71.0
|
||||
|
||||
```bash
|
||||
gbrain check-update # now resolves against real releases
|
||||
gbrain self-upgrade # now actually downloads a binary
|
||||
```
|
||||
|
||||
Or browse https://github.com/garrytan/gbrain/releases for organized per-version notes.
|
||||
|
||||
### For contributors
|
||||
|
||||
`docs/RELEASING.md` gains the release-publication section; `scripts/changelog-entry.sh` extracts a version's CHANGELOG section (used for release notes — keep entries under the standard `## [X.Y.Z.W]` headers and they publish verbatim). The workflow keeps all actions SHA-pinned, tightens top-level permissions to `contents: read` with write scoped to the release job only, and env-binds all interpolations.
|
||||
|
||||
Contributed by @time-attack (#3573, closing #3521).
|
||||
|
||||
## [0.42.70.0] - 2026-08-01
|
||||
|
||||
**Community fix wave two: 18 contributed fixes. The headline: several things you asked gbrain to do were being quietly ignored — and now they aren't.**
|
||||
|
||||
**`--brain` now actually routes.** The documented `gbrain query "X" --brain media-team` parsed the flag and then ran against your host brain anyway. It now routes to the named brain, and an unknown brain name fails loudly instead of silently answering from the wrong database.
|
||||
|
||||
**`sync --dry-run` no longer touches anything.** A dry run could pull from the remote and — if your sync strategy had changed — delete indexed pages before the "dry run" early-return was reached. Previews are now read-only, full stop.
|
||||
|
||||
**`apply-migrations --yes` applies.** It previously warned that your schema was behind and then printed "All migrations up to date" with exit 0. If you have wedged brains that upgrade never healed, this was why.
|
||||
|
||||
**Links between your pages resolve the way you write them.** Dir-qualified wikilinks with raw Obsidian names (`[[wiki/entities/AI 3.0]]`) now resolve to the sync-slugified page; references in non-whitelisted directories are no longer silently dropped; and a scan bug that could add an edge to a *parent* page you never referenced was caught in the wave's composite review and fixed before shipping.
|
||||
|
||||
**Windows and self-hosters.** Markdown files keep LF endings so frontmatter parsers stop mis-reading on Windows checkouts; the archive-crawler path gate no longer denies every real Windows path (and no longer fail-opens on NTFS case-insensitivity); a chat-synopsis tier that was hardcoded to one provider now follows your configured models; vector search asks the index for as many candidates as it was told to consider.
|
||||
|
||||
**Quieter, more honest infrastructure.** `serve --http` no longer leaves an orphan holding the database lock after Ctrl-C; a minion child that fails to launch settles immediately instead of hanging its slot; doctor gains checks for content-hash duplicates, undeclared database-only pages, stale heartbeats, and a tamper-evident manifest for the skills directory; federated reads respect per-source isolation settings in two more paths; and the security docs were rewritten to describe fixes without cataloguing attack surface.
|
||||
gbrain upgrade
|
||||
gbrain extract --stale # re-extracts links under the fixed resolver
|
||||
gbrain doctor # includes the new silent-failure checks
|
||||
```
|
||||
|
||||
If your brain uses `link_resolution.global_basename` and was populated before this release, a small number of superseded `wikilink_basename` edges can linger beside their newer typed replacements after re-extraction (edge writes are append-only by design). `gbrain reconcile-links` cleans them up; they are harmless to queries that dedup on target.
|
||||
|
||||
### For contributors
|
||||
|
||||
The composite review of this wave (two independent max-effort review passes over the combined branch) caught two interaction defects that per-PR review could not: the ungated bare-path scanner reading inside wikilink spans, and an extraction watermark set to a date that same-day stamps would already outrun. Both were fixed in the wave with discriminating tests. One reviewed-and-approved PR was deliberately held out: it conflicts semantically with its author's own sibling PR in this wave, and choosing between their two path-resolution mechanisms is the author's call.
|
||||
|
||||
Contributed by @time-attack (#3618, #3085, #3539, #3576, #3533, #3453, #3457, #3560, #3161), @daragao3 (#3619, #3536, #3517, #3578), @paul-0320 (#3613, #3564), @cvillarroel2 (#3678), @mamedov (#3624), @dialthewolff (#3550).
|
||||
|
||||
## [0.42.69.0] - 2026-08-01
|
||||
|
||||
**A community fix wave: 22 contributed fixes, most of them for work your brain was quietly not doing.**
|
||||
|
||||
The theme of this release is silent failure. A nightly cycle that reported `ok` while extracting nothing. An `embed` run that left a whole page unsearchable because one chunk in it failed, then exited 0. A health metric that recommended the same step forever because it counted one thing and the fix measured another. None of these looked broken from the outside, which is exactly why they lasted.
|
||||
|
||||
**If you run a local or non-Anthropic model, atom extraction was doing nothing.** With a cost cap set, any model absent from the pricing tables made the first work item hard-fail, which latched a budget flag and skipped every remaining item — while the phase still reported success. Local models (`ollama`, `llama-server`) now price at $0, because local inference costs electricity rather than tokens, so their caps stay enforceable. Genuinely unpriced paid providers still skip, but loudly now instead of silently.
|
||||
|
||||
**`gbrain embed` no longer lets one bad chunk darken an entire page,** and it exits non-zero when embeddings actually fail. If you have a cron wrapping `gbrain embed`, a brain holding permanently un-embeddable content will now turn that cron red. That is the intended change — it was previously green while silently incomplete.
|
||||
|
||||
**Non-Latin and diacritic names now survive mention extraction.** The by-mention tokenizer matched ASCII letters and digits only, so `Đà Nẵng` shredded into one- and two-character fragments and never matched anything. Names in Vietnamese, and any script outside ASCII, are now tokenized properly.
|
||||
|
||||
**Self-hosted embedding backends work.** Fixed-dimension OpenAI-compatible servers that reject an explicit `dimensions` parameter no longer get sent one when the requested width already matches the model's native width. A vector search on the embedded database also now asks the index for as many candidates as it was told to consider, instead of silently truncating the pool to the driver default.
|
||||
|
||||
**Multi-source brains route correctly in two more places.** A programmatic `sync_brain` call now syncs the source it was handed rather than the global default, and entity slug resolution keeps its path separators instead of flattening `people/alice-example` into an id no page can hold.
|
||||
|
||||
**Safer default on a destructive migration.** Submitting the type-unification job without an explicit `apply` now previews instead of applying. If you have that command in a runbook, add `"apply":true` — the playbooks and README were updated to show it.
|
||||
|
||||
Also: interrupted imports keep their tail instead of losing progress below the next 100-file boundary; `gbrain init --help` prints its own help instead of a stub; `doctor` stops reporting Windows drive paths as missing files under WSL and bounds its embedding health probe instead of retrying a permanent auth failure three times; cycle lock-release and stamp-write failures are visible instead of swallowed; and references to a `gbrain install` command that never existed are gone from the docs.
|
||||
|
||||
### To take advantage of v0.42.69.0
|
||||
|
||||
```bash
|
||||
gbrain upgrade # or: bun install -g gbrain@0.42.69.0
|
||||
gbrain doctor # confirms the health metric now converges
|
||||
gbrain embed --stale # exits non-zero if anything is genuinely un-embeddable
|
||||
```
|
||||
|
||||
If you use a local chat model for the nightly cycle, re-run it once and check that atoms actually land:
|
||||
|
||||
```bash
|
||||
gbrain dream --json | jq '.phases[] | select(.name=="extract_atoms")'
|
||||
```
|
||||
|
||||
If you have `gbrain jobs submit unify-types` in a runbook or script, add `"apply":true` to its `--params` or it will now preview only.
|
||||
|
||||
### For contributors
|
||||
|
||||
Two defects existed only in the *combination* of otherwise-sound fixes, and were caught by reviewing the composed branch rather than the individual changes:
|
||||
|
||||
- `isModelPriceable` was introduced with a test asserting `llama-server` is unpriced, while a second fix in the same wave priced `llama-server` at $0. Together the assertion inverted. Reconciled by using a genuinely unpriced provider in the test and pinning the positive case: free local providers are priceable at $0, so their caps stay enforced.
|
||||
- The type-unification default flipped to dry-run, but three agent-facing playbooks still presented a bare submit as the apply step. Because a second fix in the same wave also edited one of those files, each change looked self-consistent alone. Skills ship downstream via the skillpack, so this would have propagated a playbook whose apply step silently did nothing.
|
||||
|
||||
One reviewed fix was deliberately held back: extending the inline subagent drain to Postgres composes badly with this wave's minion connection-recovery work, since the drain calls the same queue operations without the new recovery path and can strand a child job in a per-run queue no worker will claim.
|
||||
|
||||
Contributed by @alexey-metaengage (#3652), @time-attack (#3568, #3572, #3567, #3555, #3523, #3144, #3574, #3532, #3545), @brettdavies (#3552, #3553), @mattchronicle (#3364), @rayers (#3589), @zenspam (#3699), @awilhite (#3691), @Grimnoth (#3541), @georgell-ceo (#3634), @Vyacheslav-Zakharov (#3631), @Kyzcreig (#3585), @HammerTech-Z (#3581), @cfeddersen (#3563).
|
||||
|
||||
## [0.42.68.1] - 2026-07-30
|
||||
|
||||
**If you run `gbrain reindex-frontmatter` or `gbrain backfill` on the default embedded database, they now work. Until this release both failed every time, after waiting 30 seconds.**
|
||||
|
||||
The embedded database allows one process at a time, and holds a lock to enforce it. These two commands opened a second connection to the same database from inside the process that already held that lock, then waited for a lock that could never be released — because the thing holding it was the waiting process itself. The wait ran its full 30 seconds and the command exited with an error naming a blocking process that was, in fact, itself. Both commands now reuse the connection that is already open.
|
||||
|
||||
Nothing changes for brains on Postgres, where a second connection was always allowed.
|
||||
|
||||
## To take advantage of v0.42.68.1
|
||||
|
||||
Nothing to undo — the commands failed without writing anything. Just run whichever you needed:
|
||||
```bash
|
||||
gbrain reindex-frontmatter
|
||||
```
|
||||
|
||||
## [0.42.67.0] - 2026-07-28
|
||||
|
||||
**If you develop GBrain on Windows, the test and check commands now actually run. Until this release they were quietly doing almost nothing.**
|
||||
|
||||
`bun run test`, `bun run verify`, `bun run ci:local` and `bun run test:e2e` all hand off to shell scripts, and on Windows that hand-off was broken in two separate places. The commands did not stop with an obvious error. They reported a result, so a run could look finished when barely any of the checks had actually inspected anything. On a clean Windows clone, `bun run verify` got 1 check to pass and 31 to fail. It now gets 25 to pass and 7 to fail, and none of the 7 are caused by this change.
|
||||
|
||||
The first problem was line endings. Git for Windows installs with `core.autocrlf=true`, which rewrites shell scripts to Windows line endings when you clone or check out. Bash refuses to run those, so a script died on its second line before doing any work. The scripts stored in the repository were always correct; only the copy on your disk was wrong. A new `.gitattributes` pins every `.sh` file to Unix line endings at checkout, no matter how your Git is configured.
|
||||
|
||||
The second problem was how the checks were started. Thirty three of them pointed straight at a `.sh` file. On macOS and Linux the shell reads the `#!/usr/bin/env bash` line at the top of the script and runs it correctly. Bun on Windows does not do that, so those commands failed the moment they were called. They now go through `bash` explicitly, the same way the other eleven were already written.
|
||||
|
||||
Nothing changes for macOS and Linux. No stored file content moves, and no check behaves differently on those platforms.
|
||||
|
||||
## To take advantage of v0.42.67.0
|
||||
|
||||
Only Windows contributors need to do anything, and only once. `.gitattributes` applies at checkout time, so shell scripts already sitting on your disk keep their old line endings until you refresh them.
|
||||
|
||||
1. **Refresh the working copy** from the repository root:
|
||||
```bash
|
||||
git rm --cached -r . -q
|
||||
git reset --hard
|
||||
```
|
||||
2. **Confirm bash can read the scripts:**
|
||||
```bash
|
||||
bash -n scripts/run-unit-parallel.sh
|
||||
```
|
||||
Silence means it worked. `$'\r': command not found` means step 1 did not take effect.
|
||||
3. **Run the gate:**
|
||||
```bash
|
||||
bun run verify
|
||||
```
|
||||
|
||||
### Itemized changes
|
||||
|
||||
- New root `.gitattributes` pins `*.sh text eol=lf`, so shell scripts check out with Unix line endings regardless of the contributor's `core.autocrlf` setting. All 59 tracked `.sh` files were already stored with Unix endings, so `git add --renormalize .` reports nothing to do and no stored content changes.
|
||||
- `package.json` now routes the remaining 33 `.sh` check commands through `bash`, matching the 11 that already did. Every tracked `.sh` file carries a bash shebang (52 `#!/usr/bin/env bash` and 7 `#!/bin/bash`), so the treatment is uniform across all of them.
|
||||
- The five `scripts/*.ts` entries still run under bun and are untouched.
|
||||
- `CONTRIBUTING.md` gains a Windows section covering the one-time working-copy refresh and the `bash scripts/<name>.sh` convention for new checks.
|
||||
- `docs/TESTING.md` records how the test commands dispatch through bash, and notes that three tree-walking checks plus `typecheck` can exceed the 120s per-check cap on Windows while passing on Linux and macOS.
|
||||
|
||||
## [0.42.66.1] - 2026-07-27
|
||||
|
||||
### Fixed
|
||||
|
||||
- `gbrain doctor` now treats embedding columns wider than pgvector's HNSW limit as healthy exact-scan configurations instead of prescribing an index PostgreSQL cannot build.
|
||||
- Local CI now passes an empty Docker mount list correctly and compiles the embedded-WASM smoke binary from container-local storage on Docker Desktop.
|
||||
|
||||
## [0.42.66.0] - 2026-07-24
|
||||
|
||||
**54 verified fixes from the community backlog: background enrichment stops wasting money on dead pages, autopilot stops killing its own healthy runs, and search respects your settings.**
|
||||
|
||||
This release is the second big sweep through the open pull-request backlog, with every change reviewed and tested individually before merging. The theme is trust in the background machinery. The overnight "dream" cycle now remembers which pages produced nothing and stops re-reading them every night, meters its small-model calls against your spend caps, and keeps claim proposals from silently overwriting each other. Long consolidation runs get a 30-minute deadline instead of being killed at 10 minutes mid-work. A wedged server boot now releases its database lock instead of blocking every later command.
|
||||
|
||||
Search behaves the way you configured it: the recency-decay setting now actually applies to hybrid search, a local `list_pages` call returns as many rows as you asked for, and when a listing is cut short it says so instead of looking complete. Slack conversation exports parse cleanly, with an optional AI fallback for formats the parser does not know.
|
||||
|
||||
New provider recipes: DashScope reranking, OpenRouter reranking, and a claude-cli recipe for dispatching subagents through the gateway.
|
||||
|
||||
## To take advantage of v0.42.66.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. One schema migration ships in this release (v125, take-proposal idempotency); it is idempotent and needs no manual action.
|
||||
|
||||
1. **Upgrade and verify:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
2. **If `gbrain doctor` warns about a partial migration**, run the orchestrator manually:
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Dream cycle, takes, and spend control
|
||||
|
||||
- Pages whose extraction yields zero claims are memoized, so the cycle stops re-spending on them every night. (#2514, #3319, contributed by @ivandebot)
|
||||
- Zero-yield pages are tombstoned so `extract_atoms` stops rediscovering them. (#2144, #3304, contributed by @ChenyqThu)
|
||||
- `extract_atoms` Haiku calls are metered against the cost gate. (#2371, #3329, contributed by @TheRealMrSystem)
|
||||
- `extract_atoms` stamps concepts so `synthesize_concepts` has material to work with. (#2123, #3308, contributed by @ChenyqThu)
|
||||
- `extract_facts` requires a live backing page, not just a non-NULL entity slug. (#2497, #3321, contributed by @javieraldape)
|
||||
- Multi-claim pages keep every proposal instead of only the first (migration v125 makes the idempotency key per claim). (#3297, contributed by @rp-agent-bot)
|
||||
- Superseding a take now queries the active row first. (#3275, contributed by @arisgysel-design)
|
||||
- Takes keyword search matches words inside long claims via `word_similarity`. (#3267)
|
||||
- Dream-generated orphan pages stay scoped to their source. (#2368, #3344, contributed by @snvtac)
|
||||
- Drift detection is wired into the dream cycle, report-only for now. (#2653, #3317)
|
||||
|
||||
#### Autopilot, jobs, and serve
|
||||
|
||||
- Full consolidation cycles get a 30-minute timeout floor; lighter dispatches keep the interval-derived budget. (#2852, #3338, contributed by @sanchalr)
|
||||
- The cron wrapper exports `~/.bun/bin` onto PATH so autopilot survives minimal environments. (#2013, #3305, contributed by @klampatech)
|
||||
- Dead or cancelled jobs no longer block idempotent re-submission. (#2253, #3306, contributed by @rafaelreis-r)
|
||||
- Contextual reindex jobs get a default timeout. (#2611, #3323, contributed by @spiky02plateau)
|
||||
- Onboarding stops repeating the same auto-remediation within a single run. (#2854, #3342, contributed by @sanchalr)
|
||||
- A wedged `gbrain serve` boot hits a readiness deadline and releases the PGLite lock. (#3335)
|
||||
|
||||
#### Search, retrieval, and health
|
||||
|
||||
- The recency-decay config is honored on the hybrid search path. (#2386, #3312, contributed by @rwbaker)
|
||||
- `list_pages` honors explicit limits for local callers, warns on remote clamping, and threads `offset`. (#2591, #3322, contributed by @deacon-botdoctor)
|
||||
- Truncated `list_pages` results say so instead of silently capping. (#2865, #3341, contributed by @paul-0320)
|
||||
- Negative metrics no longer invert trajectory regression signals. (#2621, #3324, contributed by @morluto)
|
||||
- Per-chunk synopsis generation in contextual retrieval is concurrency-bounded. (#2628, #3326, contributed by @spiky02plateau)
|
||||
- Graph health metrics count `entity` pages. (#2639, #3330, contributed by @tylr-r)
|
||||
|
||||
#### Ingestion, extraction, and links
|
||||
|
||||
- Conversation parsing gains an opt-in LLM fallback for unknown formats. (#2247, #3371, contributed by @danwiggins)
|
||||
- Normalized Slack markdown parses into conversations. (#3289, #3372, contributed by @danwiggins)
|
||||
- Conversation backfill outcomes are durable, so completed pages skip on the next run. (#3293, #3373, contributed by @danwiggins)
|
||||
- Reference-style wikilinks are recognized during extraction. (#2071, #3303, contributed by @mzkarami)
|
||||
- `[[wikilink]]` frontmatter values resolve via global basename lookup. (#2406, #3313, contributed by @spiky02plateau)
|
||||
- Incremental push syncs extract links. (#2850, #3337, contributed by @patentsong)
|
||||
- `<think>` reasoning tags in extractor output are handled. (#2559, #3318, contributed by @qaz8545355)
|
||||
- Tiktoken special tokens no longer crash code-chunker token estimates. (#2453, #3315, contributed by @Jiglet)
|
||||
- Source config stops re-wrapping into a growing JSON string scalar. (#2829, #3334, contributed by @1alessio)
|
||||
|
||||
#### Providers and recipes
|
||||
|
||||
- DashScope reranking recipe (DashScope serves a plural `/reranks` endpoint under its compatible API). (#2644, #3328, contributed by @YiconZiwei)
|
||||
- OpenRouter reranking touchpoint. (#2164, #3302, contributed by @Hippityy)
|
||||
- claude-cli recipe for native gateway-based subagent dispatch. (#2277, #3310, contributed by @brettdavies)
|
||||
- Prefixed model IDs work on the openai-compatible embedding-dimensions path. (#2325, #3309, contributed by @noetherly)
|
||||
- Embeddings stamp the gateway-resolved model in `content_chunks.model`, not the compiled default. (#2846, #3343, contributed by @SailorJoe6)
|
||||
- Bun-on-Windows write-through EEXIST fixed, non-Anthropic `--max-cost` pricing works, dream pages excluded from enrich. (#2407, #3316, contributed by @nguyenchiviet)
|
||||
- Supabase signed URLs prepend `/storage/v1`. (#2565, #3320, contributed by @danwiggins)
|
||||
|
||||
#### Sources, auth, and multi-brain
|
||||
|
||||
- Federated-source pages are visible to `get_page`, `list_pages`, `resolve_slugs`, and no-grant MCP callers. (#3242, #3301)
|
||||
- Admin-gated rescope surface for DCR clients stuck on a default scope. (#3299)
|
||||
- `whoami` exposes OAuth source grants. (#3279, #3332, contributed by @boundless-forest)
|
||||
- Thin-client `--source` maps onto `source_id` for remote-routed operations. (#3086)
|
||||
|
||||
#### CLI, doctor, and init
|
||||
|
||||
- `gbrain doctor` stops claiming "Brain is at target" when the target is unreachable. (#2151, #3339, contributed by @brettdavies)
|
||||
- Doctor gains a raw-source persistence guarantee for synthesized pages, warn-only for now. (#3300)
|
||||
- Doctor timeline labels disambiguate entity coverage from the brain-score component. (#2298, #3073, contributed by @TurgutKural)
|
||||
- Unknown `gbrain init` flags are rejected before migrations run. (#2201, #3307, contributed by @caioribeiroclw-pixel)
|
||||
- The init soul-audit hint points at the conversational skill, not a nonexistent CLI verb. (#2486, #3314, contributed by @SeanGearin)
|
||||
- `--force` retry escapes completed migration-ledger entries. (#2616, #3325, contributed by @spiky02plateau)
|
||||
- PGLite data-dir lock contention gets a clear error message. (#2658, #3336, contributed by @zaycruz)
|
||||
- Frontmatter validation derives slugs from the brain root, not the absolute path. (#2340, #3311, contributed by @alessioalionco)
|
||||
|
||||
#### For contributors
|
||||
|
||||
- Docker network isolation guidance for co-located self-hosted Postgres. (#3270, #3331)
|
||||
- `CLAUDE.local.md` / `AGENTS.local.md` are gitignored. (#3290, contributed by @igbymyboy)
|
||||
- The hybrid-reranker integration test isolates `GBRAIN_HOME`. (#1527, #3327, contributed by @Willisbest)
|
||||
- Test-shard scripts capture the real exit code before watchdog teardown in the no-timeout fallback. (#2864, #3340, contributed by @paul-0320)
|
||||
|
||||
## [0.42.65.0] - 2026-07-23
|
||||
|
||||
**A large maintenance release: 93 verified fixes and small features merged since v0.42.64.0, most of them community contributions.**
|
||||
|
||||
If you use gbrain day to day, this release makes the boring parts trustworthy. Importing and syncing notes is safer: a failed pull no longer pretends everything is up to date, imported pages are read back after writing to confirm they landed, and a page with real content can no longer be silently overwritten by an empty one. Search answers get better inputs: the think command now picks excerpts that actually match your question, and results respect your federated source settings. Background enrichment (the "dream" cycle) wastes less money and retries properly when an AI provider is down. Spending caps now fail closed, so a billing hiccup can never turn into an uncapped spend. And `gbrain doctor` is quieter, with several false alarms removed and real problems (like an embedding backlog with no worker running) now flagged.
|
||||
|
||||
More AI providers work out of the box, including OpenRouter prompt caching, MiniMax and Zhipu GLM recipes, Ollama Matryoshka embedding dimensions, and llama-server batch limits.
|
||||
|
||||
## To take advantage of v0.42.65.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. No new schema migrations ship in this release.
|
||||
|
||||
1. **Upgrade and verify:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
2. **If `gbrain doctor` reports new findings after upgrading,** that is the quieter, more accurate check set working as intended. Each finding names its fix.
|
||||
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Security
|
||||
|
||||
- MCP source scoping for remote callers got a hardening pass, so agent-facing connections stay confined to the sources they were granted. (#2881, contributed by @spinsirr)
|
||||
- Paid MCP spend accounting is now atomic and fails closed, and resolver spend is recorded before a cap error is raised, so caps cannot be raced past or undercounted. (#3203, #3204, contributed by @caterpillarC15)
|
||||
- The OAuth token endpoint rate limit on the HTTP server is now configurable via env for deployments behind shared IPs. (#3114, contributed by @time-attack)
|
||||
- `WWW-Authenticate` responses now carry `resource_metadata` per the MCP spec and RFC 9728, so conforming clients can discover the auth server. (#1410, contributed by @rayers)
|
||||
|
||||
#### Search, retrieval, and think
|
||||
|
||||
- `think` selects query-relevant excerpts instead of generic ones. (#3197, contributed by @Y0lan)
|
||||
- Unqualified local CLI `search`/`query` now honors `sources.config.federated` read visibility. (#2561, #3141, contributed by @time-attack)
|
||||
- Email citation metadata is projected into search results. (#2873, contributed by @amtagrwl)
|
||||
- The `think` Gaps section renders once instead of twice. (#1662, contributed by @howwohmm)
|
||||
- Fuzzy entity lookup threads the caller's source scope and skips soft-deleted entities. (#1508, contributed by @tim404x)
|
||||
- `code-def` surfaces method, constructor, field, and struct definitions, not just top-level symbols. (#1628, contributed by @rayers)
|
||||
- Briefing pages are excluded from their own Brain Pulse salience. (#1202, contributed by @rwbaker)
|
||||
- Reranker calls with missing auth are classified as configuration errors before falling back. (#2059, #3139, contributed by @time-attack)
|
||||
|
||||
#### Import, sync, and ingestion
|
||||
|
||||
- A failed git pull with zero imports reports `partial (pull_failed)` instead of `up_to_date`. (#3068, #3253, contributed by @Masashi-Ono0611)
|
||||
- Imports run a post-write read-back verification with a durable ingest-log record. (#2869, contributed by @Andredsouza1984)
|
||||
- `put` refuses to overwrite a non-empty page with empty content. (#2708, contributed by @symmetric-matthew)
|
||||
- `putPage` restores soft-deleted rows instead of colliding with them. (#2779, contributed by @RerankerGuo)
|
||||
- Mixed-case slugs are normalized before chunk upsert, ending duplicate-chunk churn. (#430, #3143, contributed by @time-attack)
|
||||
- Imports fall back to the body H1 for the title when frontmatter lacks `title:`. (#2446, #3072, contributed by @time-attack)
|
||||
- YAML comments inside the frontmatter fence are no longer treated as markdown headings. (#3225, #3247, contributed by @Masashi-Ono0611)
|
||||
- Write-through guards case-insensitive filesystem collisions before the atomic write. (#2831, #3119, contributed by @time-attack)
|
||||
- Path-qualified wikilinks outside the known directory pattern resolve on the DB/put_page path. (#2866, contributed by @paul-0320)
|
||||
- CJK slugs are supported in the slug registry and dream-cycle summary slugs. (#782, #738, #3083, contributed by @time-attack)
|
||||
- Three ingest/sync/serve singleton fixes: page-type round-trip, deleted-slug embed noise, and a stateless width guard. (#3140, contributed by @time-attack)
|
||||
- Sync honors the `embedding_disabled` sentinel as an implicit `--no-embed`. (#2879, contributed by @gawievanblerk)
|
||||
- Verified sync head sentinels are cleared correctly. (#2734, contributed by @symmetric-matthew)
|
||||
- Resumed syncs report the pinned commit they actually landed on. (#3202, contributed by @caterpillarC15)
|
||||
- The expected `discover_git_root` probe failure stays off stderr. (#3232, contributed by @Masashi-Ono0611)
|
||||
- `extract --stale` runs the real resolver so basename resolution reaches stale pages, and clears pre-version-bump pages. (#2576, #2717, contributed by @paul-0320; #1791, contributed by @Nazim22)
|
||||
- Oversized code chunks are capped so they stay embeddable, and code-chunk metadata survives re-embeds. (#1675, contributed by @lubosxyz; #769, #1232, contributed by @rayers)
|
||||
|
||||
#### Background cycle, dream, and facts
|
||||
|
||||
- Path-derived dream sources are stamped, and the engine closes cleanly on autopilot shutdown. (#3178, contributed by @time-attack)
|
||||
- All-provider-failed atom drains propagate so durable jobs retry instead of silently dropping work. (#3218, #3248, contributed by @Masashi-Ono0611)
|
||||
- Atom extraction raises `maxTokens` and case-normalizes `atom_type` for Gemini models. (#3211, contributed by @alexey-metaengage)
|
||||
- The conversation extractor gates anonymous-speaker self-attribution instead of guessing. (#3228, contributed by @asenkovskiy)
|
||||
- Incremental dream extraction stamps its watermark so re-runs stop reprocessing. (#2636, #3115, contributed by @time-attack)
|
||||
- `dream --dry-run --json` keeps stdout clean of embed summaries. (#394, #3109, contributed by @time-attack)
|
||||
- Synthesized dream pages require a self-contained opening summary. (#2770, contributed by @Masashi-Ono0611)
|
||||
- PGLite inline synth subagent drains complete, and `lint` gains `--exclude`. (#2699, #2649, #3162, contributed by @time-attack)
|
||||
- Live context reads the documented "P1 Today" heading form with plain checkbox tasks, matching the daily-task-manager skill's output format. (#2186, #3124, contributed by @time-attack)
|
||||
- Queued AI jobs refresh gateway config at execution time instead of using a stale snapshot. (#2125, contributed by @maxpetrusenkoagent)
|
||||
- `brainstorm`/`propose_takes` honor configured models: cost preview uses the configured model, the judge reads its config key, provider probes are skipped when unneeded, and page projection is narrowed. (#3120, contributed by @time-attack)
|
||||
- Backlog hardening wave: x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog handling, and pooler direct-URL routing. (#3165, contributed by @time-attack)
|
||||
- `skillopt` emits `proposed.md` in no-mutate mode. (#2635, #3182, contributed by @time-attack)
|
||||
- Nightly quality probe enable path and conversation-parser probe are wired up. (#2629, #2630, #3094, contributed by @time-attack)
|
||||
|
||||
#### Doctor, health, and maintenance
|
||||
|
||||
- New safe maintenance automation with a shared orphan-exclusion policy, so routine cleanup runs without risking linked content. (#3015, #3023, contributed by @time-attack)
|
||||
- `orphan_ratio` excludes the chronicle volume under `life/events/`. (#2264, #3214, contributed by @asenkovskiy)
|
||||
- `brain_score` orphan/timeline components use the orphans-audit linkable scope. (#3155, contributed by @time-attack)
|
||||
- Entity timeline coverage is measured separately from whole-brain density. (#2761, contributed by @TurgutKural)
|
||||
- Doctor flags embed backfills queued with no worker running. (#2696, contributed by @javieraldape)
|
||||
- Two doctor false-positive/timeout fixes: the drift walk skips `node_modules`, and the bare-tweet check skips inline code and cited lines. (#1772, contributed by @sonlndv)
|
||||
- A dead `llm_fallback_enabled` recommendation is dropped from conversation format coverage. (#1903, contributed by @ElliotDrel)
|
||||
- Skill triggers with CRLF line endings parse on Windows. (#1149, contributed by @samporter-31)
|
||||
- Onboard check names are registered in doctor categories, ending unknown-check warnings, and onboard-check remediations survive the `--apply --auto` path. (#3075, #3097, contributed by @time-attack)
|
||||
- Dead slug prefixes are counted by slug. (#2697, contributed by @RerankerGuo)
|
||||
- The backlinks worker defaults to check, not fix, and `check-backlinks` honors its positional directory argument. (#1853, contributed by @choomz; #3076, contributed by @time-attack)
|
||||
- Calibration resolves the owner holder via config, defaulting to `self`. (#3077, contributed by @time-attack)
|
||||
- Memory throttling on Linux reads `/proc/meminfo` MemAvailable. (#556, contributed by @chengzehsu)
|
||||
|
||||
#### AI providers and gateway
|
||||
|
||||
- OpenRouter gets family-scoped prompt caching, and query expansion works on chat-capable openai-compat recipes. (#3152, contributed by @time-attack)
|
||||
- MiniMax recipe: embedding wire-shape compat fetch plus a chat touchpoint. (#1977, #3089, contributed by @time-attack)
|
||||
- The Zhipu recipe gains a chat touchpoint so GLM subagents work. (#1157, #3084, contributed by @time-attack)
|
||||
- Tier-configured models reach the recipe allowlist, Anthropic model lists are refreshed, tier resolutions are registered, and probe labels are honest. (#2800, contributed by @p3ob7o)
|
||||
- Provider base URL config merges from the DB. (#1676, contributed by @TheLordArgus)
|
||||
- The gateway falls back to the pooler when the derived direct host is unreachable. (#1641, #3088, contributed by @time-attack)
|
||||
- Config-plane `voyage_api_key` folds into `VOYAGE_API_KEY` like the other hosted keys. (#3236, contributed by @Masashi-Ono0611)
|
||||
- The `zeroentropyai:zerank-2` reranker has a pricing entry so the budget tracker can meter it. (#3223, #3233, contributed by @Masashi-Ono0611)
|
||||
- llama-server embedding batches are capped at its 32-input request limit. (#1281, contributed by @mmekkaoui)
|
||||
- Matryoshka dimensions thread through for Qwen3-Embedding on Ollama. (#1072, contributed by @mgandal)
|
||||
- `init` seeds AI options from env on cold install, and `whoami` reports the stdio transport. (#3091, contributed by @time-attack)
|
||||
- The `models` dispatch subcommand reads its first argument correctly. (#1428, contributed by @BenjaminDSmithy)
|
||||
- Synopsis generation tail-truncates document text for small-model chat handlers. (#1427, contributed by @BenjaminDSmithy)
|
||||
- The contradiction judge token cap is raised for thinking models. (#3210, contributed by @alexey-metaengage)
|
||||
|
||||
#### Schema, migrations, and storage engines
|
||||
|
||||
- Engine migration counts and surfaces per-page copy failures instead of silently advancing. (#3241, contributed by @Masashi-Ono0611)
|
||||
- Invalid `CONCURRENTLY`-build index remnants are dropped without a DO block. (#3191, contributed by @Masashi-Ono0611)
|
||||
- Unsupported large-dimension HNSW indexes are skipped instead of failing schema setup. (#1734, #3080, contributed by @time-attack)
|
||||
- The v0.32.2 migration dirty-check scopes to targeted sources and surfaces failed phase detail. (#3093, contributed by @time-attack)
|
||||
- Schema packs merge the full `extends` chain and `borrow_from` into the resolved manifest. (#1749, #3181, contributed by @time-attack)
|
||||
- The schema-pack stats catch-all is narrowed so masked errors surface instead of fake zero-page counts. (#2466, #3133, contributed by @time-attack)
|
||||
- Bundled schema-pack inspection reports the pack actually shipped in the binary, and minion subagent auth resolves through config. (#3110, contributed by @time-attack)
|
||||
- PGLite `putPage` guards against zero-row RETURNING. (#1649, contributed by @alexhawkins)
|
||||
|
||||
#### MCP server and CLI surface
|
||||
|
||||
- `list_pages` rows include `source_id`. (#3209, contributed by @alexey-metaengage)
|
||||
- Running CLI commands while `gbrain serve` (MCP) holds the brain now notifies about the conflict instead of failing confusingly. (#3243, contributed by @fdefitte)
|
||||
- The OpenClaw plugin manifest entry is declared so the plugin loads. (#2551, #3185, contributed by @time-attack)
|
||||
|
||||
#### For contributors
|
||||
|
||||
- CI scanner roots are normalized on macOS. (#3198, contributed by @caterpillarC15)
|
||||
- CI shard timeout raised to 22 minutes plus a delta-assert reporter leak test. (#3231, contributed by @time-attack)
|
||||
- E2E suite hardening: flaky tests, no-op assertions, and cross-test coupling removed. (#1704, contributed by @auroracapital)
|
||||
- `mechanical.test.ts` isolates `$HOME` so the E2E suite stops clobbering user config. (#434, contributed by @lloydarmbrust)
|
||||
- The lint code-fence-wrap detector and fixer regex now agree. (#1597, contributed by @chungty)
|
||||
- README project links for OpenClaw and Hermes are corrected. (#1961, #3179, contributed by @time-attack)
|
||||
- A completed TODOS entry is dropped. (#3229, contributed by @Masashi-Ono0611)
|
||||
|
||||
## [0.42.64.0] - 2026-07-20
|
||||
|
||||
### Fixed
|
||||
|
||||
- Confidential OAuth clients can now revoke access tokens through the standard revocation endpoint when client secrets are stored as hashes. Invalid credentials fail closed, malformed or mixed authentication is rejected, backend failures remain retryable, and discovery metadata accurately advertises supported authentication methods.
|
||||
|
||||
No schema migrations.
|
||||
## [0.42.63.0] - 2026-07-20
|
||||
|
||||
**Schema commands now open the local brain you actually configured.**
|
||||
|
||||
If your PGLite brain lives at a custom path, commands such as `gbrain schema stats` previously ignored that path and could inspect the default brain instead. That made a healthy configured brain look empty or report the wrong schema counts. Schema commands now use the same complete database configuration as the rest of GBrain. PostgreSQL behavior is unchanged, and no migration is required.
|
||||
|
||||
### How to use it
|
||||
|
||||
Upgrade, then run the schema command normally:
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain schema stats --json
|
||||
```
|
||||
|
||||
The reported page and type counts now come from the `database_path` in `~/.gbrain/config.json` when the engine is PGLite.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Fixed
|
||||
- **Schema CLI commands preserve configured PGLite paths.** Engine construction and connection now receive the canonical complete engine configuration, including both `database_path` and `database_url` where applicable.
|
||||
- **CLI tests are isolated from ambient database URLs.** Schema subprocess tests explicitly clear inherited PostgreSQL URL variables, and a persistent-PGLite regression test proves `schema stats` reads the configured database rather than the default brain.
|
||||
|
||||
## [0.42.62.0] - 2026-07-17
|
||||
|
||||
**If your brain holds more than one source, everything now lands in the right one. Link extraction, timeline extraction, background cycles, and webhook captures used to quietly file some of their output under the default source; all of those paths now carry the correct source identity. Background agent jobs got tougher too: a failed database reconnect can no longer wedge the engine, and workers recover from dropped connections instead of crash-looping. If you run the admin dashboard behind a reverse proxy, the live activity panel finally connects. Long agent conversations cost less because repeated context is reused between turns on Anthropic calls. Local LiteLLM proxies work out of the box. Nested sources scan correctly again instead of reporting zero files. And the project's automated checks now include dependency vulnerability scanning, static code-security analysis, and signed provenance for release builds. Thirty merged changes in all, the largest batch to date, each one reviewed and verified against the live codebase before landing.**
|
||||
|
||||
## To take advantage of v0.42.62.0
|
||||
|
||||
`gbrain upgrade`. No new schema migrations.
|
||||
|
||||
1. **Multi-source brains:** run `gbrain extract all` once (or let the next cycle do it) so previously mis-scoped link and timeline rows are regenerated under the right source.
|
||||
2. **If you serve the admin dashboard behind a reverse proxy,** hard-refresh it once after upgrading; Live Activity should connect.
|
||||
3. **Verify:**
|
||||
```bash
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
4. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor`.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Fixed
|
||||
- **Source identity threaded through write paths.** Filesystem link/timeline extraction (`src/commands/extract.ts`), the cycle extract phase, and ingest capture now stamp the resolved source id instead of defaulting to `default`, with fail-closed validation on externally supplied ids. (#1522, #1747, #1503 via #2920; absorbs #1719, contributed by @seungsu)
|
||||
- **`reconnect()` is build-then-swap.** The new pool is validated before replacing the old one, so a failed rebuild restores the previous connection instead of leaving `_sql` null. (#1593 follow-up via #1906, contributed by @rayers)
|
||||
- **Minion worker reconnects after promote-time connection loss** instead of crash-looping. (#1491 class via #2025, contributed by @maxpetrusenkoagent)
|
||||
- **Admin Live Activity works behind reverse proxies.** The EventSource now sends credentials so strict-cookie sessions survive the proxy hop. (#912 via #1560, contributed by @flamerged)
|
||||
- **Stats exclude soft-deleted pages** from visible counts on both engines; destructive-removal counts stay all-inclusive. (#2235, contributed by @xd-Neji)
|
||||
- **LiteLLM recipes declare chat and expansion touchpoints,** so the subagent loop no longer swaps to Anthropic and fails without an Anthropic key. (#2207 via #2208, contributed by @brettdavies)
|
||||
- **Rolling prompt-cache on the direct SDK path.** Growing conversations place rolling cache breakpoints (two, within the four-marker budget), cutting repeat-token cost on multi-turn Anthropic tool loops. (#2740 via #2771, contributed by @Masashi-Ono0611)
|
||||
- **Nested sources scan again.** `sources audit` had one inverted prune check (descending into node_modules while reporting 0 files). (#2678, contributed by @ikamal97)
|
||||
- **Import and sync agree on metafiles.** The import walker now skips the same structural metafiles sync skips. (#345 via #2315, contributed by @ElliotDrel)
|
||||
- **Frontmatter scans respect git excludes** via a shared git-visible-files helper. (#2462, contributed by @kubi-dev)
|
||||
- **Sync renames are crash-safe** (per-file failures recorded instead of aborting the run) and **zero-change syncs still bump `last_sync_at`** so freshness reporting stops lying. (#2402, contributed by @supportswift; #2335, contributed by @lost9999)
|
||||
- **Facts survive one-shot CLI runs.** Facts-absorb work is enqueued as durable minion jobs instead of dying with the process exit drain; fence paths are source-scoped. (#2104, contributed by @reghar-bot)
|
||||
- **Takes reads are source-scoped, `gbrain calibration` is reachable, outputs are BigInt-safe.** (#2035 and the takes slice of #2200 via #2892, takeover of #2452, contributed by @spinsirr)
|
||||
- **CLI answers honestly.** `config get` reads both config planes with provenance, `sources archive` is idempotent, help text matches real subcommands, doctor recommendations name commands that exist. (#2120, #2792, #1175, #1123, #2451 via #2918)
|
||||
- **PGLite init failures name plausible causes for your platform** instead of blaming a macOS-specific bug everywhere, and non-Error crashes print their message instead of `[object Object]`. (#2674 class via #2891)
|
||||
- **YAML comments inside frontmatter parse.** `#` lines inside a closed fence are comments, not headings; no more false MISSING_CLOSE. (#2152 via #2153, contributed by @brettdavies)
|
||||
- **Conversation facts read the raw transcript sidecar** and recognize plain `Speaker A:` lines. (#1897 via #1898, contributed by @ElliotDrel)
|
||||
- **`get_timeline` exposes date-window filters** (#2604 via #2694, contributed by @RerankerGuo) and **`query` since/until filter on effective date,** not updated_at (#1520 via #1706, contributed by @mvanhorn).
|
||||
- **Windows serve watchdog works** via a signal-0 liveness probe instead of a POSIX-only process listing. (#2049, contributed by @abyss-node)
|
||||
- **Doctor probes route through the active engine** (no false pgvector/jsonb warnings on PGLite; #1513 via #1183, contributed by @duncanclaw) and **a disabled retrieval reflex reads as intentional** (#2459, contributed by @eloe).
|
||||
- **Cross-platform installs.** The postinstall hook is a real bun script, not POSIX shell that failed on Windows. (#1486 via #1554, contributed by @Sanjays2402)
|
||||
- **Agent-bound auth clients.** `auth register-client` gains the `--bound-*` flags the submit_agent gate requires. (#1945, #1971 via #1976, contributed by @mzkarami)
|
||||
|
||||
#### Added
|
||||
- **Security automation in the project's checks:** scheduled OSV dependency scanning, Semgrep static analysis on every PR (non-blocking initially), and build-provenance attestations wired into the release workflow. (#2182, #2142, #2272 via #2917)
|
||||
- **`provider_chat_options` config passthrough** to the gateway, e.g. disabling thinking mode per provider or model. (#2577 via #2857)
|
||||
- **Docs:** macOS 26.x PGLite workaround and native Postgres setup guide. (#1671, contributed by @roysaurav)
|
||||
|
||||
#### Internal
|
||||
- release.yml runs `verify` before building. (#2222 via #2243, contributed by @mzkarami)
|
||||
- Regenerated llms bundle after the docs merge. (#2893)
|
||||
|
||||
## [0.42.61.0] - 2026-07-16
|
||||
|
||||
**If gbrain's background daemon dies hard, a restart now takes over right away instead of waiting minutes for a stale lock to expire. Re-processing the same content no longer piles up near-duplicate knowledge atoms. On large brains, the takes bootstrap finally works through the whole corpus instead of re-scanning the same newest pages every run. And `gbrain schema use` can now activate the schema packs gbrain actually ships — including the install default — instead of just one hardcoded name. Cost tracking also learns the newest Claude models, so spend on them is metered instead of invisible.**
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Fixed
|
||||
- **Autopilot recovers immediately from a crashed daemon.** The stale-lock check verifies whether the lock-holding process is still alive instead of relying on a fixed age window — a hard-killed autopilot no longer delays restarts, and the age check alone can no longer displace a busy, live one. (#477, contributed by @vinsew)
|
||||
- **Atom extraction stops minting duplicate atoms across runs.** Atom slugs are now deterministic (source-dated, canonical slugging, content-hashed suffix), so re-extracting the same content upserts instead of creating a near-duplicate under a new run-date path, and titles that truncate mid-word no longer produce trailing-dash slug variants. Pre-existing duplicates are not re-created but remain until cleaned up (an `atoms consolidate` command is tracked as a follow-up). (#2482, contributed by @joelwp)
|
||||
- **Takes bootstrap works through the whole corpus.** Bootstrap runs skip pages that already have takes, so brains larger than the per-run page cap make forward progress instead of rescanning the newest slice and re-spending extraction budget. `--include-covered` restores the old behavior. (#2638, contributed by @p3ob7o)
|
||||
- **`gbrain schema use` can activate the core bundled packs.** The command resolved only one hardcoded pack name; it now resolves through the bundled-pack registry, so the recommended and v2 base packs (including the install default) can be selected. (#1707, contributed by @mvanhorn)
|
||||
- **Budget tracking prices Sonnet 5 and Fable 5.** The canonical chat-pricing table adds the newest Claude models at standard list rates (time-limited introductory discounts are deliberately not modeled, so early Sonnet 5 spend reads slightly conservative), removing the no-pricing blind spot in cost telemetry and budget metering. (#2799, contributed by @p3ob7o)
|
||||
|
||||
#### Added
|
||||
- **Inline `[Source: ..., YYYY-MM-DD]` citations become timeline entries.** Both the filesystem extract path and the auto-timeline write path recognize the citation convention gbrain’s own quality guidance recommends, with idempotent re-extraction. (#2524, contributed by @pabloglzg)
|
||||
- **Schema packs extend atom-extraction page discovery.** For packs that declare the `extract_atoms` phase, the manifest’s `extractable` flag now unions with the legacy page-type list (synthesis outputs stay excluded, so concepts never feed back into atom extraction). (#2615, contributed by @p3ob7o)
|
||||
- **Book-mirror two-column pages are generated as HTML tables** with top alignment instead of markdown pipe tables, which broke on multi-paragraph cells in most renderers. (#2270)
|
||||
|
||||
#### Internal
|
||||
- Gateway tool-schema conversion extracted into a tested helper so the regression test exercises the exact code path production uses. (#2063, contributed by @maxpetrusenkoagent)
|
||||
- Reference docs synced for the v0.42.59.0 fixes (engine/testing entries). (#2798, contributed by @time-attack)
|
||||
|
||||
### To take advantage of v0.42.61.0
|
||||
|
||||
`gbrain upgrade`. No new schema migrations.
|
||||
|
||||
1. **Heads-up on extraction scope:** if your active schema pack declares the `extract_atoms` phase, page types the pack marks `extractable` now feed atom extraction alongside the legacy list — the first cycle after upgrading may process page types (notes, emails, slack) it previously skipped. Per-run page and budget caps still apply; check `gbrain search stats` / budget output if you watch spend closely.
|
||||
2. **If takes bootstrap seemed stuck** on a large brain, re-run it — each run now covers new pages.
|
||||
3. **Verify:**
|
||||
```bash
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
4. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor`.
|
||||
|
||||
## [0.42.60.0] - 2026-07-16
|
||||
|
||||
**Eleven verified community fixes: Windows brains no longer risk losing subdirectory pages on a full sync, agent tool loops on non-Anthropic providers survive interruption instead of dead-lettering, multi-source brains get two source-isolation gaps closed, and the search cache stops leaking results across exclude policies. Every fix was reproduced and reviewed against master before landing.**
|
||||
|
||||
### Fixed
|
||||
- **Windows: `sync --full` no longer deletes subdirectory pages.** A path-separator mismatch made every subdirectory page look stale during full-sync reconcile, so a routine full sync could delete them. Paths are now normalized before comparison, and a mass-delete safety valve blocks any reconcile that would remove most of a source's pages. (#2828, #2836, contributed by @1alessio)
|
||||
- **Gateway tool loops on non-Anthropic providers are reliable across resume.** Tool-result turns are persisted as they happen, interrupted jobs reconcile dangling tool calls on resume instead of dead-lettering with unbalanced-transcript errors, `Date` values in tool outputs no longer crash serialization, DeepSeek reasoning-only replies are read correctly instead of as empty, and `openrouter_api_key` in config reaches the gateway. (#2820, consolidating community fixes #2062, #2065, #2257, #2274, #2336, #2487, #2491, #2572, #2614, #2617, #2806; contributed by @time-attack and the original PR authors)
|
||||
- **Claude 5 models get output-token headroom.** Thinking-default models no longer have long answers silently truncated by the old 4096-token default output cap; Claude 5 chat calls now default to 32000 output tokens (16000 for `think`). Other providers keep their existing caps, so smaller-limit providers are unaffected. (#2820)
|
||||
- **Bulk import survives huge fence-less files.** The markdown lexer is skipped when a page contains no code fences, removing an out-of-memory crash on large tables and notes during bulk import. (#2437, #2440, contributed by @irresi)
|
||||
- **`file_list` no longer crashes on Postgres brains over MCP.** BIGINT file sizes are normalized before JSON serialization; the CLI files listing gets the same fix. (#472, contributed by @vinsew)
|
||||
- **`gbrain config set auto_chronicle true` works as documented.** The Life Chronicle config keys (and `takes.bootstrap_enabled`) are registered, so the documented enable commands stop being rejected as unknown keys. (#2632, contributed by @p3ob7o)
|
||||
- **Orphan reports skip generated corpus roots.** `raw/`, `atoms/`, and `skills/` no longer inflate the orphan ratio by default; `--include-pseudo` still shows everything. (#2068, contributed by @mgunnin)
|
||||
|
||||
### Security
|
||||
- **The search cache honors your hard-exclude policy.** Cached search results are now keyed on the effective hard-exclude/include slug-prefix policy, so a process with `GBRAIN_SEARCH_EXCLUDE` set can never be served cached rows written under a different policy — and vice versa. (#2825, #2885)
|
||||
- **Take-writes are source-scoped.** When a source resolves (via `--source`, `GBRAIN_SOURCE`, or the dotfile chain), CLI take commands look pages up within that source instead of first-match-by-slug, closing a cross-source write path on brains where the same slug exists in multiple sources. Brains without a resolvable source keep the previous lookup. (#2684, #2698, contributed by @RerankerGuo)
|
||||
- **Image pages land in the right source.** Imported images are stamped with the syncing source (and their auto-links stay within it) instead of always landing in `default`. (#2706, #2718, contributed by @RerankerGuo)
|
||||
- **The admin bootstrap token no longer prints to a non-terminal stream.** The one-time token is withheld when its output stream is a pipe, log, or CI capture instead of an interactive terminal. (#2625, contributed by @irresi)
|
||||
|
||||
### Internal
|
||||
- Pinned embedding dimensions in a doctor test to eliminate a shard-order flake in CI. (#2801, contributed by @p3ob7o)
|
||||
|
||||
### To take advantage of v0.42.60.0
|
||||
|
||||
`gbrain upgrade`. No new schema migrations.
|
||||
|
||||
1. **Windows users with git-synced sources:** re-run `gbrain sync --full` once after upgrading — if a pre-upgrade sync deleted subdirectory pages, they re-import from the repo.
|
||||
2. **Your first search after upgrading may be a cache miss** (the cache key now includes the exclude policy). Speeds return to normal as the cache refills within its TTL.
|
||||
3. **If agent jobs previously dead-lettered** with unbalanced tool-call transcript errors on OpenAI-compatible providers, retry them with `gbrain jobs retry <id>` — resume now reconciles the transcript.
|
||||
4. **Verify:**
|
||||
```bash
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
5. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor`.
|
||||
|
||||
## [0.42.59.0] - 2026-07-13
|
||||
|
||||
**Five community-reported fixes, each reproduced and verified before/after on both engines (PGLite + real Postgres): an upgrade wedge that locked pre-v121 brains out of migrations, two data-integrity holes in engine migration, silent deletion of facts containing pipe characters, confidently-wrong entity attribution on ambiguous names, and tightened source-scope enforcement in `think`.**
|
||||
|
||||
### Fixed
|
||||
- **Existing brains below schema v121 can upgrade again.** Brains created before v0.42.56.0 could get stuck in a loop where every command (including `apply-migrations`) failed with `column "event_page_id" does not exist` — the migration that adds the column could never run. The startup bootstrap now adds the forward-referenced column first; migration v121 still owns the FK and indexes. Re-running is idempotent, and already-wedged brains heal on the next command. (#2724, #2735, contributed by @time-attack)
|
||||
- **`gbrain migrate --to` no longer fails on multi-source brains.** The source catalog is copied before pages, so the first page no longer dies on a foreign-key violation. Source rows migrate with full fidelity (paths, sync state, config). (#2677, #2736, contributed by @time-attack)
|
||||
- **Migration resume checkpoints are target-aware.** An interrupted migration to one target no longer convinces a later migration to a *different* target that most pages are "already done" (which silently shorted the new target). A checkpoint for another destination is discarded and the run starts fresh; no connection strings or credentials are written to manifests or logs. (#2677, #2736, contributed by @time-attack)
|
||||
- **Facts containing `|` characters survive reconciliation.** The facts fence rendered literal pipes escaped but re-parsed rows by splitting on every pipe, so any fact whose text contained a `|` was silently deleted from the DB on the next extract-facts cycle. Render→parse is now symmetric (pipes, backslashes, and empty cells verified round-trip). The takes fence shares the parser and gets the same fix. (#2726, #2738, contributed by @time-attack)
|
||||
- **Ambiguous entity names quarantine instead of guessing.** A bare first name shared by two people, or a company name sharing a generic token (e.g. "… Capital") with another company, used to resolve confidently to the wrong entity — misattributed facts are invisible and expensive to repair. Bare names now resolve only when exactly one canonical candidate exists; low-specificity fuzzy matches fall through to the guarded holding path (a held fact is recoverable; a misattributed one isn't). Explicit slugs, full names, unique bare names, and close typos still resolve. Trade-off: heavier typos on short names may now hold instead of resolving. (#2723, #2737, contributed by @time-attack)
|
||||
|
||||
### Security
|
||||
- **`think` now applies the caller's source scope across all of its internal retrieval.** Hybrid page retrieval, takes keyword/vector retrieval, and graph traversal all honor scalar and federated source scope, matching the isolation the rest of the read surface already enforces. Part of the #2200 tracking work. (#2739, contributed by @time-attack)
|
||||
|
||||
### To take advantage of v0.42.59.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. No new schema migrations ship in this release (v121/v122 shipped with v0.42.56.0).
|
||||
|
||||
1. **If your brain was stuck below schema v121** (every command printed a schema-probe warning), just upgrade and run any command — the brain heals and migrates to current on first connect. If `gbrain doctor` still complains:
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
2. **Verify:**
|
||||
```bash
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
3. **If a previously-resolving shorthand name now files under a holding page**, that's the new ambiguity quarantine working as intended — add an alias or use the full name/slug for entities you want bare shorthand to hit.
|
||||
4. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
## [0.42.58.0] - 2026-07-06
|
||||
|
||||
**gbrain now runs cleanly on the stack you already have — a local Ollama box, a self-hosted LiteLLM proxy, llama.cpp's llama-server, or gbrain running as a Claude Code MCP subprocess — instead of silently degrading or hard-failing when you're not on a raw OpenAI/Anthropic key.** A provider-agnostic plumbing pass across the AI gateway: environment handling, base-URL normalization, and embedding-dimension validation all stop tripping on the non-frontier-vendor setups that used to fail without a clear signal.
|
||||
|
||||
@@ -67,6 +67,19 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
|
||||
@@ -11,6 +11,34 @@ bun test
|
||||
|
||||
Requires Bun 1.0+.
|
||||
|
||||
### Windows
|
||||
|
||||
`bun run test`, `verify`, `ci:local` and `test:e2e` all dispatch through bash, so
|
||||
the shell scripts under `scripts/` must be checked out with Unix line endings.
|
||||
The root `.gitattributes` pins `*.sh text eol=lf`, which overrides the
|
||||
`core.autocrlf=true` that Git for Windows installs by default. A fresh clone is
|
||||
correct with no extra steps.
|
||||
|
||||
`.gitattributes` pins `*.md text eol=lf` for the same reason. The frontmatter
|
||||
readers anchor on a `---` fence followed by a Unix line ending, so a CRLF
|
||||
checkout makes a well-formed document parse as having no frontmatter. That
|
||||
failure is silent: no error, the field just comes back empty.
|
||||
|
||||
If you cloned before either pin existed, your working copy still has the old
|
||||
Windows line endings. Bash will fail with `$'\r': command not found`, and
|
||||
frontmatter will read as absent. Refresh it once, from the repository root:
|
||||
|
||||
```bash
|
||||
git rm --cached -r . -q
|
||||
git reset --hard
|
||||
bash -n scripts/run-unit-parallel.sh # silence means bash can read the scripts
|
||||
git ls-files --eol -- '*.md' | grep -c w/crlf # 0 means Markdown is clean
|
||||
```
|
||||
|
||||
Every `check:*` entry in `package.json` invokes its script as `bash scripts/<name>.sh`
|
||||
rather than relying on the shebang, because bun on Windows cannot exec a `.sh`
|
||||
directly. Keep that prefix when you add a new shell-script check.
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
@@ -163,6 +191,14 @@ host port with `GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
Fail-closed selector: an unmapped `src/` change runs all 29 E2E files. Hand-tune
|
||||
narrower mappings via `scripts/e2e-test-map.ts`.
|
||||
|
||||
### PR-side security checks
|
||||
|
||||
Besides the test gate, PRs may trigger three security workflows: Semgrep CE
|
||||
SAST (every PR — **advisory/non-blocking** while the baseline is tuned, so a
|
||||
Semgrep finding won't fail your PR), OSV-Scanner (only when `package.json` or
|
||||
`bun.lock` change), and actionlint (only when `.github/workflows/**` change).
|
||||
See `SECURITY.md` → "Automated security scanning" for details.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
|
||||
@@ -16,6 +16,13 @@ If you fetched this file by URL without cloning yet, the companion files live at
|
||||
|
||||
## Step 1: Install GBrain
|
||||
|
||||
> **NEVER install from the npm registry.** GBrain is not distributed on npm; the npm
|
||||
> package named `gbrain` is an unrelated package. Do NOT run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` (note the missing `github:` prefix — that's the trap). The only
|
||||
> supported sources are `github:garrytan/gbrain` and a git clone, exactly as shown below.
|
||||
> If an unrelated npm install is already present, remove it first
|
||||
> (`npm uninstall -g gbrain` / `bun remove -g gbrain`); `gbrain doctor` also detects this.
|
||||
|
||||
Default path (Bun is required — gbrain is a Bun + TypeScript runtime):
|
||||
|
||||
```bash
|
||||
|
||||
@@ -65,14 +65,24 @@ This is the difference between a search engine and a brain. Search finds the pag
|
||||
|
||||
## Install
|
||||
|
||||
> [!WARNING]
|
||||
> **GBrain is NOT distributed on npm.** The npm package named `gbrain` is an unrelated
|
||||
> package with no connection to this project. Do not run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` — you'll get something else, and it can shadow the real binary on
|
||||
> your PATH. Install and upgrade ONLY via the documented paths below
|
||||
> (`bun install -g github:garrytan/gbrain`, or `git clone` + `bun install && bun link`).
|
||||
> If you already ran the npm install by mistake: `npm uninstall -g gbrain` /
|
||||
> `bun remove -g gbrain`, then reinstall from GitHub. `gbrain doctor` detects a
|
||||
> shadowing npm install and prints the fix.
|
||||
|
||||
GBrain is designed to be installed and operated by an AI agent. The fastest path is to have your agent do it for you. The CLI and MCP paths below are for people who want to wire it up themselves.
|
||||
|
||||
### Have your agent install it (recommended)
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
@@ -208,7 +218,7 @@ Most personal-knowledge tools force one fixed layout: their idea of "notes" + "p
|
||||
**gbrain doesn't have a fixed layout.** It ships with bundled schema packs and lets you author your own when none fit:
|
||||
|
||||
- **`gbrain-base-v2`** (default as of v0.41.22) — 15-type DRY/MECE canonical taxonomy (14 canonical + `note` catch-all): `person`, `company`, `media`, `tweet`, `social-digest`, `analysis`, `atom`, `concept`, `source`, `deal`, `email`, `slack`, `writing`, `project`, `note`. Subtypes/format/origin pushed to frontmatter. The taxonomy that responds to issue #1479.
|
||||
- **`gbrain-base`** (legacy, v0.41 and earlier brains) — the original 24-type layout. Stays bundled for back-compat; brains on it can upgrade via `gbrain onboard --check --explain` → `gbrain jobs submit unify-types --allow-protected --params '{"target_pack":"gbrain-base-v2"}'`.
|
||||
- **`gbrain-base`** (legacy, v0.41 and earlier brains) — the original 24-type layout. Stays bundled for back-compat; brains on it can upgrade via `gbrain onboard --check --explain` → `gbrain jobs submit unify-types --allow-protected --params '{"target_pack":"gbrain-base-v2","apply":true}'` (omit `"apply":true` for a dry-run preview — that is the default).
|
||||
- **`gbrain-recommended`** — extends `gbrain-base` with the 13 additional directories from `docs/GBRAIN_RECOMMENDED_SCHEMA.md` (source, place, trip, conversation, personal, civic, project, etc.). Activate with `gbrain schema use gbrain-recommended`.
|
||||
- **Your own pack** — `gbrain schema detect` clusters your actual filesystem into proposed types, `gbrain schema suggest` runs an LLM pass over them, and `gbrain schema review-candidates --apply` promotes the ones you like. Three commands and the brain knows your shape. Authoring a successor pack (declares `migration_from:` so existing brains can opt in): see [`docs/architecture/pack-upgrade-mechanism.md`](docs/architecture/pack-upgrade-mechanism.md).
|
||||
|
||||
@@ -260,6 +270,24 @@ The whole loop is described in [`docs/architecture/topologies.md`](docs/architec
|
||||
|
||||
**Job queue (Minions).** BullMQ-shaped, Postgres-native job queue. Durable subagents (LLM tool loops that survive crashes via two-phase pending→done persistence), shell jobs with audit, child jobs with cascading timeouts, rate leases for outbound providers, attachments via S3/Supabase storage. Replaces "spawn subagent as fire-and-forget Promise" with something that recovers from anything.
|
||||
|
||||
**Non-English brains (FTS language config).** The Postgres full-text search tokenizer is configurable via `GBRAIN_FTS_LANGUAGE`. Defaults to `english`. Set it to any text-search configuration that exists in your Postgres instance:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese # uses built-in portuguese stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=spanish # built-in spanish stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br # custom config (e.g. unaccent + portuguese)
|
||||
```
|
||||
|
||||
List available configs: `psql -c "SELECT cfgname FROM pg_ts_config"`. Both the **query side** (`websearch_to_tsquery`) and the **write side** (the trigger functions that populate `pages.search_vector` and `content_chunks.search_vector`) honor `GBRAIN_FTS_LANGUAGE`. On first install (or upgrade), the `configurable_fts_language` schema migration reads the env var and creates trigger functions in the configured language; subsequent inserts/updates tokenize using that setting. To change language on a brain that has already run the migration, use the dedicated CLI command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command is idempotent (re-running with the same language is a no-op for vector content) and uses the same recreate-and-backfill primitives as the migration. For accent-insensitive Portuguese (`pt_br`), see [docs/guides/multi-language-fts.md](docs/guides/multi-language-fts.md) for the `unaccent` + portuguese stemmer recipe.
|
||||
|
||||
**43 curated skills.** Routing lives in [`skills/RESOLVER.md`](skills/RESOLVER.md). Covers signal capture, ingest (idea / media / meeting), enrichment, querying, brain ops, citation fixing, daily task management, cron scheduling, reports, voice, soul audit, skill creation, eval framework, and migrations. Skills are markdown files (tool-agnostic), packaged as a single skillpack the installer drops into your agent workspace.
|
||||
|
||||
**Eval framework.** `gbrain eval longmemeval` runs the public [LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval) benchmark against your hybrid retrieval. `gbrain eval export` + `gbrain eval replay` capture real queries and replay them against code changes (set `GBRAIN_CONTRIBUTOR_MODE=1`). `gbrain eval cross-modal` cross-checks an output against the task using three different-provider frontier models. `gbrain eval retrieval-quality` runs NamedThingBench, which hard-gates the named-thing retrieval families (title-substring, alias-synonym, generic-to-named, multi-chunk-dilution) so a regression in "find the page this query names" fails CI loudly. Full methodology in [`docs/eval/SEARCH_MODE_METHODOLOGY.md`](docs/eval/SEARCH_MODE_METHODOLOGY.md).
|
||||
@@ -291,6 +319,8 @@ Data flowing into the brain. Each integration is a recipe — markdown + setup h
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**`gbrain init --pglite` crashes on macOS 26.x (Tahoe)?** PGLite's embedded WASM engine is incompatible with macOS 26.x on Apple Silicon. The fix is to use native Homebrew PostgreSQL + pgvector instead. Full step-by-step setup in [`docs/INSTALL.md` — Troubleshooting: PGLite crashes on macOS 26.x](docs/INSTALL.md#pglite-crashes-on-macos-26x-tahoe).
|
||||
|
||||
**`gbrain import` fails with `expected N dimensions, not M`?** Run `gbrain doctor`. It will print the exact `gbrain config set ...` or `gbrain retrieval-upgrade` command to repair the mismatch. You should not need to delete `~/.gbrain`. Fresh `gbrain init --pglite` auto-detects your embedding provider from API keys in your environment: set `OPENAI_API_KEY` (or `ZEROENTROPY_API_KEY` / `VOYAGE_API_KEY`) before running init, or pass `--embedding-model <provider>:<model>` explicitly. With multiple keys set, init fires an interactive picker. In non-TTY contexts (CI, Docker) with no keys, init exits 1 with a paste-ready setup hint; pass `--no-embedding` to defer setup until runtime. See [`docs/integrations/embedding-providers.md`](docs/integrations/embedding-providers.md) for the full provider matrix and [`docs/operations/headless-install.md`](docs/operations/headless-install.md) for Docker/CI sequencing.
|
||||
|
||||
**Hourly cron sync keeps timing out on a federated brain?** v0.41.13.0 ships
|
||||
|
||||
+54
-23
@@ -8,17 +8,44 @@ on GitHub.
|
||||
|
||||
Do not open a public issue for security vulnerabilities.
|
||||
|
||||
## Automated security scanning
|
||||
|
||||
CI runs three automated security checks alongside secret scanning (Gitleaks):
|
||||
|
||||
- **Dependency vulnerabilities** — OSV-Scanner
|
||||
(`.github/workflows/osv-scanner.yml`) runs weekly and on any PR that touches
|
||||
`package.json` or `bun.lock`.
|
||||
- **Static analysis (SAST)** — Semgrep CE (`.github/workflows/semgrep.yml`)
|
||||
runs on every PR and weekly. It is currently **advisory (non-blocking)**
|
||||
while the finding baseline is tuned; the graduation path to a blocking check
|
||||
is documented in the workflow file.
|
||||
- **Release binary provenance** — release builds
|
||||
(`.github/workflows/release.yml`) attest each compiled binary with
|
||||
[GitHub artifact attestations](https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations).
|
||||
Verify a downloaded release binary with:
|
||||
|
||||
```bash
|
||||
gh attestation verify ./gbrain-darwin-arm64 -R garrytan/gbrain
|
||||
gh attestation verify ./gbrain-linux-x64 -R garrytan/gbrain
|
||||
```
|
||||
|
||||
All security workflows use SHA-pinned actions and least-privilege permissions,
|
||||
enforced structurally by actionlint on every workflow change.
|
||||
|
||||
## Remote MCP Security
|
||||
|
||||
### ⚠️ Do NOT use open OAuth client registration for remote MCP
|
||||
### Keep dynamic client registration disabled unless explicitly needed
|
||||
|
||||
If you deploy GBrain's MCP server behind an HTTP wrapper with OAuth 2.1
|
||||
support, **never allow unauthenticated client registration**. An attacker
|
||||
who discovers your server URL can:
|
||||
GBrain disables Dynamic Client Registration (DCR) by default. Keep that
|
||||
default for internet-reachable deployments and pre-register trusted clients
|
||||
with operator-approved scopes and source access. Enabling DCR lets network
|
||||
callers create OAuth client records, so use it only when the deployment's
|
||||
trust model requires self-service registration and browser approval remains
|
||||
part of the authorization flow.
|
||||
|
||||
1. Register a new OAuth client via `POST /register`
|
||||
2. Use `client_credentials` grant to obtain a bearer token
|
||||
3. Access all brain data via the MCP tools
|
||||
Do not enable `--enable-dcr-insecure` on an untrusted network. That option is
|
||||
reserved for deployments that intentionally allow self-registered
|
||||
machine-to-machine clients without browser approval.
|
||||
|
||||
### Recommended: `gbrain serve --http`
|
||||
|
||||
@@ -80,12 +107,10 @@ Auth methods (`--token-endpoint-auth-method`):
|
||||
- `none` — public PKCE-only client (no secret minted; ChatGPT custom
|
||||
connector, Claude Code, Cursor)
|
||||
|
||||
The validator rejects unknown methods at the registration boundary, and
|
||||
the same gate applies to the admin endpoint `POST /admin/api/register-client`
|
||||
and the DCR `POST /register` path. Pre-v0.41.3 the CLI hard-coded
|
||||
`redirect_uris = []` and `token_endpoint_auth_method = NULL`, forcing
|
||||
operators to UPDATE `oauth_clients` rows by hand to make claude.ai work
|
||||
without `--enable-dcr`. That footgun is gone.
|
||||
The same validator applies to CLI, admin, and DCR registration paths, so
|
||||
unknown authentication methods are rejected consistently. Browser-based
|
||||
clients can be configured entirely through the supported CLI flags; operators
|
||||
do not need to edit OAuth database rows by hand.
|
||||
|
||||
### DCR consent default (v0.42.55+)
|
||||
|
||||
@@ -135,6 +160,18 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### Docker network isolation (self-hosted Postgres)
|
||||
|
||||
OAuth and source scoping enforce isolation on the `serve --http` path only.
|
||||
Raw Postgres reachability bypasses both: a container that shares Docker's
|
||||
default `bridge` network with the brain's Postgres can open a direct DB
|
||||
session without any token and read every source. Put the brain's Postgres on
|
||||
a user-defined Docker network with nothing untrusted on it, publish its port
|
||||
loopback-only (if at all), and never put `DATABASE_URL` or a Postgres
|
||||
password in untrusted agent containers — those should reach the brain
|
||||
exclusively via OAuth against `serve --http`. Full operator checklist:
|
||||
[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
@@ -150,16 +187,10 @@ When the request `Origin` matches the allowlist, the server echoes it
|
||||
back in `Access-Control-Allow-Origin` (with `Vary: Origin`). Otherwise no
|
||||
CORS header is sent and the browser blocks the request.
|
||||
|
||||
**v0.41.3:** the same allowlist now gates every OAuth endpoint (`/mcp`,
|
||||
`/token`, `/authorize`, `/register`, `/revoke`). Pre-v0.41.3 these used
|
||||
default-wide-open `cors()` middleware, leaking
|
||||
`Access-Control-Allow-Origin: *` on every response — any web origin could
|
||||
complete a token exchange from a logged-in operator's browser. The CORS
|
||||
preflight handler in the legacy bearer transport was also asymmetric
|
||||
(actual-request path correctly default-deny, but OPTIONS preflight leaked
|
||||
`Access-Control-Allow-Methods` + `Access-Control-Allow-Headers` to every
|
||||
Origin); both are now consolidated through a single allowlist-gated path.
|
||||
A startup stderr WARN fires when `--bind 0.0.0.0` is set without
|
||||
The same allowlist gates the complete MCP and OAuth HTTP surface. Actual
|
||||
requests and browser preflight requests use one allowlist-gated policy, so
|
||||
unlisted origins receive no cross-origin authorization. A startup stderr
|
||||
warning fires when `--bind 0.0.0.0` is set without
|
||||
`GBRAIN_HTTP_CORS_ORIGIN`, surfacing the default-deny posture before the
|
||||
first request.
|
||||
|
||||
|
||||
@@ -23,23 +23,102 @@ and the scope record at `~/.gstack/projects/garrytan-gbrain/ceo-plans/2026-06-12
|
||||
`src/core/verbs/entity-card.ts` open-threads assembly + a new schema table
|
||||
(additive — the card field already exists, so this is a quality upgrade, not
|
||||
a contract change).
|
||||
## v0.42.67.0 follow-ups (Windows build tooling)
|
||||
|
||||
Filed as follow-ups from v0.42.67.0 (`.gitattributes` LF pin for `*.sh` +
|
||||
`bash` prefix on the 33 `package.json` check commands). Both items are newly
|
||||
observable: before that release these checks never executed on Windows at all,
|
||||
so nothing about their runtime was measurable.
|
||||
|
||||
- [ ] **P2 — three guard scripts exceed the 120s `run-verify-parallel.sh` cap on Windows.**
|
||||
With the dispatch fixed, `bun run verify` on Windows gets 25 passes and 7 failures, and
|
||||
`check:privacy`, `check:test-names` and `check:test-isolation` are timeouts rather than
|
||||
real failures (they pass on Linux and macOS well inside the cap). They walk the tree with
|
||||
per-file shell loops, which is far slower under Windows process creation. Either raise the
|
||||
cap for these three, or replace the per-file loop with a single `grep -r` pass. Same cap
|
||||
swallows `typecheck`, though standalone `bun run typecheck` exits 0.
|
||||
- [ ] **P3 — `check:wasm` cannot create its `node_modules` symlink on Windows.**
|
||||
`scripts/check-wasm-embedded.sh` fails with `ln: failed to create symbolic link
|
||||
'/tmp/gbrain-wasm-check.XXXX/node_modules': No such file or directory`. Unprivileged
|
||||
Windows accounts cannot create symlinks without developer mode. Consider a junction, a
|
||||
copy, or skipping the check with a clear message when symlink creation is unavailable.
|
||||
|
||||
## community fix-wave follow-ups (filed v0.42.60.0)
|
||||
|
||||
- [x] **P2 — cherry-pick #2112's uncovered doctor.ts hunk.** Fix-wave A (#2820) superseded
|
||||
most of #2112 but not its `checkSubagentCapability` fix (check explicit `models.subagent`
|
||||
before `models.tier.subagent`). Implemented: `checkSubagentCapability` now resolves
|
||||
`models.subagent` before tier/default fallbacks and has regression coverage.
|
||||
|
||||
## v0.42.59.0 follow-ups (five-fix rollup #2735–#2739)
|
||||
|
||||
Filed as follow-ups from v0.42.59.0 (bootstrap probe for
|
||||
`timeline_entries.event_page_id`, migrate-engine source catalog + target-aware
|
||||
resume, entity-resolution quarantine, escape-aware fence cells, think gather
|
||||
source scope).
|
||||
|
||||
- [ ] **P2 — schema-bootstrap-coverage strip block never exercises `timeline_entries.event_page_id`.**
|
||||
The guard's pre-migration-brain simulation (the strip DDL in
|
||||
`test/schema-bootstrap-coverage.test.ts`) has no
|
||||
`ALTER TABLE timeline_entries DROP COLUMN IF EXISTS event_page_id` (or FK drop), so the
|
||||
coverage entry added for the v121 forward reference is vacuous — the probe never fires
|
||||
under that harness. The real regression guard lives in `test/bootstrap.test.ts` (which
|
||||
does drop → re-bootstrap → assert). Add the DROP statements to the strip block so the
|
||||
coverage test genuinely exercises its own entry.
|
||||
- [ ] **P2 — extract-facts reconcile still wipes-then-reinserts when the parse emitted MALFORMED warnings.**
|
||||
`runExtractFacts` (`src/core/cycle/extract-facts.ts`) deletes a page's facts and
|
||||
reinserts from the parsed fence even when `parseFactsFence` surfaced
|
||||
`FACTS_TABLE_MALFORMED` warnings — any future parse defect becomes a deletion vector
|
||||
(rows the parser failed to read get wiped with nothing to reinsert). Consider
|
||||
skip-wipe-on-warnings: treat a warning-bearing parse as non-authoritative for that page
|
||||
(skip the wipe, surface a warn), mirroring the empty-fence legacy-row guard's posture.
|
||||
- [ ] **P3 — bare-name resolution quarantines even on an exact unique match when prefix siblings exist.**
|
||||
With pages `companies/acme` + `companies/acme-labs`, a bare `"Acme"` yields two
|
||||
`findPrefixCandidates` rows, so `tryUnambiguousPrefixExpansion` declines — even though
|
||||
`companies/acme` is an exact `dir/token` slug match (and may be a unique exact title
|
||||
match). That's an unambiguity signal being wasted. Consider promoting an exact
|
||||
`dir/token` (or exact-title) hit above the sibling-count check in
|
||||
`src/core/entities/resolve.ts`.
|
||||
- [ ] **P2 — `scripts/run-verify-parallel.sh` no-gtimeout fallback reports the watchdog's exit code, not the check's.**
|
||||
In the fallback branch, `rc=$?` is captured after `wait "$cap_pid"` (the killed
|
||||
sleep-watchdog, rc=143) rather than after `wait "$pid"` (the actual check) — on a Mac
|
||||
without coreutils every check false-fails with rc=143. Capture `rc` from `wait "$pid"`
|
||||
first, then reap the watchdog.
|
||||
- [ ] **P3 — same-target migrate resume with `--force` still skips checkpointed pages after the wipe.**
|
||||
`gbrain migrate --to <engine> --force` wipes the target's pages, but the resume
|
||||
manifest's `completed_slugs` filter still applies, so previously-checkpointed pages are
|
||||
skipped against the now-empty target (pre-existing behavior; the v0.42.59.0 verification
|
||||
warns about it). `--force` should clear the manifest when it matches the same target.
|
||||
Where: `src/commands/migrate-engine.ts`.
|
||||
- [ ] **P2 — think residual scope gaps.** Two spots in `src/core/think/index.ts` don't yet
|
||||
inherit the caller's source scope the way the gather stage now does:
|
||||
`persistCitations` resolves citation slugs with an unscoped
|
||||
`SELECT id FROM pages WHERE slug = $1 LIMIT 1` (cross-source slug ambiguity can attach
|
||||
saved evidence to the wrong same-slug page), and the trajectory entity-resolution scalar
|
||||
is `opts.sourceId ?? 'default'` (a federated caller with `allowedSources` but no scalar
|
||||
resolves entities against `default` instead of its grant). Mirror the gather-stage
|
||||
precedence (federated array > scalar > default) at both sites.
|
||||
|
||||
## provider-agnostic follow-ups (filed v0.42.58.0)
|
||||
|
||||
Deferred from the provider-agnostic plumbing wave (#1249/#1250/#1292/#2271/#2209).
|
||||
Plan + review trail at `~/.claude/plans/system-instruction-you-are-working-keen-newell.md`.
|
||||
The eng-review + Codex outside-voice narrowed the wave to these deferrals:
|
||||
|
||||
- [ ] **P2 — Capability-aware query expansion on OpenAI-compat providers (#2372).**
|
||||
- [x] **P2 — Capability-aware query expansion on OpenAI-compat providers (#2372).**
|
||||
Expansion only runs for recipes that declare an `expansion` touchpoint, and only the
|
||||
native providers (anthropic/openai/google) do. To make expansion work on
|
||||
litellm/openrouter/groq/together/deepseek you must ADD expansion touchpoints to those
|
||||
chat-capable recipes AND add a `generateObject`→`generateText` capability fallback for
|
||||
backends without strict structured outputs. Feature-shaped; overlaps the general
|
||||
OpenAI-compat proxy story (`docs/designs/COMMUNITY_IDEAS.md`). Community PR #2373 is a
|
||||
starting point. Where: `src/core/ai/gateway.ts:expand`, recipe files, `types.ts` (ExpansionTouchpoint).
|
||||
- [ ] **P2 — LiteLLM as a chat/expansion backend.** `litellm-proxy` declares ONLY an
|
||||
starting point. Implemented by #2373 plus the DeepSeek/Groq/Together recipe wave,
|
||||
LiteLLM chat/expansion support, and the OpenRouter expansion touchpoint. Where:
|
||||
`src/core/ai/gateway.ts:expand`, recipe files, `types.ts` (ExpansionTouchpoint).
|
||||
- [x] **P2 — LiteLLM as a chat/expansion backend.** `litellm-proxy` declares ONLY an
|
||||
embedding touchpoint, so `think`/chat on LiteLLM is dead. Add chat (and expansion) so a
|
||||
LiteLLM proxy is a full LLM backend, not embedding-only. The general OpenAI-compat proxy story.
|
||||
LiteLLM proxy is a full LLM backend, not embedding-only. Implemented by #2208.
|
||||
The general OpenAI-compat proxy story.
|
||||
- [ ] **P3 — Per-model embedding dims metadata on `EmbeddingTouchpoint`.** `default_dims`
|
||||
is recipe-wide, so a recipe (ollama) can't carry different native dims per model. This
|
||||
wave added the modern ollama model NAMES + a `trust_custom_dims` passthrough (user supplies
|
||||
@@ -2247,10 +2326,25 @@ at plan time and got carved out:
|
||||
via `buildPerSourceBindings`. Document workaround: register
|
||||
source-scoped OAuth clients.
|
||||
|
||||
- [ ] **v0.41+: T20 — extends-chain merging in registry.ts.**
|
||||
`registry.ts:167` documents the gap. Implementing full child-wins
|
||||
merge cascades through every consumer of `manifest.page_types`. ~1
|
||||
day CC.
|
||||
- [x] **v0.41+: T20 — extends-chain merging in registry.ts.** DONE (#1749).
|
||||
`resolvePack` now merges parent → child (child-wins) for the six
|
||||
ingest/query-shaping fields (`page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, `takes_kinds`)
|
||||
plus `borrow_from` materialization, in `src/core/schema-pack/merge.ts`.
|
||||
The cascade was transparent (consumers already read `resolved.manifest`),
|
||||
not per-consumer. `phases`/`calibration_domains` deliberately excluded —
|
||||
see the P3 follow-up below.
|
||||
|
||||
- [ ] **P3: explicit opt-in to inherit `phases` / `calibration_domains`.**
|
||||
T20 excludes these two from the child-wins merge because they gate real
|
||||
cycle execution (`cycle.ts` `packDeclaresPhase`) and the manifest
|
||||
contract says each pack declares its own participation explicitly —
|
||||
auto-inheriting would silently make a child run cycle phases it never
|
||||
requested. Multi-level lens packs (`gbrain-everything`) therefore still
|
||||
re-declare them by hand. If that redeclaration becomes painful, add an
|
||||
explicit manifest flag (e.g. `inherit_phases: true`) so a pack author
|
||||
opts in consciously. Depends on: T20 (landed). Start in
|
||||
`src/core/schema-pack/merge.ts` (`mergeInheritedManifest`).
|
||||
|
||||
- [ ] **v0.41+: T21 — comment-preserving YAML emitter.**
|
||||
v0.40.7.0 emitter does NOT preserve comments. Authors who care
|
||||
@@ -2509,7 +2603,7 @@ contributor traps.
|
||||
|
||||
- [ ] **v0.37.x: Adopt `resolveDefaultHeaders` for Together / Groq / other attribution-bearing recipes.** v0.37.6.0's `default_headers` / `resolveDefaultHeaders` seam is generic — any recipe whose provider benefits from app-attribution headers can opt in. Together and Groq both have rankings/analytics tied to per-app headers. Add their respective attribution headers to each recipe, similar to OR's `HTTP-Referer` + `X-OpenRouter-Title`. No type-system or gateway changes needed; just `default_headers` blocks on the existing recipes plus `<PROVIDER>_REFERER` / `<PROVIDER>_TITLE` env vars in their `auth_env.optional`. Filed during v0.37.6.0 eng review as a D4 generalization opportunity.
|
||||
|
||||
- [ ] **v0.37.x: Guard cli.ts `main()` so importing `buildGatewayConfig` doesn't print help.** v0.37.6.0 exported `buildGatewayConfig` from `src/cli.ts` for test access. Importing it triggers the file's top-level `main()` which prints help to stdout during tests — functionally harmless (tests pass) but noisy. Fix: wrap `main()` in `if (import.meta.main)` so it only runs when cli.ts is the entry point, not when imported. Touches one line; trivial. Filed during v0.37.6.0 implementation.
|
||||
- [x] **v0.37.x: Guard cli.ts `main()` so importing `buildGatewayConfig` doesn't print help.** v0.37.6.0 exported `buildGatewayConfig` from `src/cli.ts` for test access. Importing it triggers the file's top-level `main()` which prints help to stdout during tests — functionally harmless (tests pass) but noisy. Fix: wrap `main()` in `if (import.meta.main)` so it only runs when cli.ts is the entry point, not when imported. Touches one line; trivial. Filed during v0.37.6.0 implementation.
|
||||
|
||||
|
||||
## v0.37.4.0 pgGraph CI scaffolding follow-ups (v0.37.x+)
|
||||
|
||||
+52
-20
@@ -13,48 +13,52 @@
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"typescript": "^5.8.3",
|
||||
"vite": "^6.3.3",
|
||||
"vite": "^6.4.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"overrides": {
|
||||
"@babel/core": "^7.29.6",
|
||||
"postcss": "^8.5.23",
|
||||
},
|
||||
"packages": {
|
||||
"@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="],
|
||||
"@babel/code-frame": ["@babel/code-frame@7.29.7", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw=="],
|
||||
|
||||
"@babel/compat-data": ["@babel/compat-data@7.29.0", "", {}, "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg=="],
|
||||
"@babel/compat-data": ["@babel/compat-data@7.29.7", "", {}, "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg=="],
|
||||
|
||||
"@babel/core": ["@babel/core@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-compilation-targets": "^7.28.6", "@babel/helper-module-transforms": "^7.28.6", "@babel/helpers": "^7.28.6", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/traverse": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA=="],
|
||||
"@babel/core": ["@babel/core@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-compilation-targets": "^7.29.7", "@babel/helper-module-transforms": "^7.29.7", "@babel/helpers": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA=="],
|
||||
|
||||
"@babel/generator": ["@babel/generator@7.29.1", "", { "dependencies": { "@babel/parser": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw=="],
|
||||
"@babel/generator": ["@babel/generator@7.29.7", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ=="],
|
||||
|
||||
"@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.28.6", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA=="],
|
||||
"@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.29.7", "", { "dependencies": { "@babel/compat-data": "^7.29.7", "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g=="],
|
||||
|
||||
"@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="],
|
||||
"@babel/helper-globals": ["@babel/helper-globals@7.29.7", "", {}, "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA=="],
|
||||
|
||||
"@babel/helper-module-imports": ["@babel/helper-module-imports@7.28.6", "", { "dependencies": { "@babel/traverse": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw=="],
|
||||
"@babel/helper-module-imports": ["@babel/helper-module-imports@7.29.7", "", { "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g=="],
|
||||
|
||||
"@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.6", "", { "dependencies": { "@babel/helper-module-imports": "^7.28.6", "@babel/helper-validator-identifier": "^7.28.5", "@babel/traverse": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA=="],
|
||||
"@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.29.7", "", { "dependencies": { "@babel/helper-module-imports": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7", "@babel/traverse": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg=="],
|
||||
|
||||
"@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.28.6", "", {}, "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug=="],
|
||||
|
||||
"@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
"@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="],
|
||||
|
||||
"@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
"@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="],
|
||||
|
||||
"@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="],
|
||||
"@babel/helper-validator-option": ["@babel/helper-validator-option@7.29.7", "", {}, "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw=="],
|
||||
|
||||
"@babel/helpers": ["@babel/helpers@7.29.2", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.29.0" } }, "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw=="],
|
||||
"@babel/helpers": ["@babel/helpers@7.29.7", "", { "dependencies": { "@babel/template": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg=="],
|
||||
|
||||
"@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="],
|
||||
"@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="],
|
||||
|
||||
"@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="],
|
||||
"@babel/template": ["@babel/template@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg=="],
|
||||
|
||||
"@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="],
|
||||
"@babel/traverse": ["@babel/traverse@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-globals": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/types": "^7.29.7", "debug": "^4.3.1" } }, "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw=="],
|
||||
|
||||
"@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
"@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
|
||||
|
||||
@@ -220,7 +224,7 @@
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
"nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="],
|
||||
|
||||
"node-releases": ["node-releases@2.0.37", "", {}, "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg=="],
|
||||
|
||||
@@ -228,7 +232,7 @@
|
||||
|
||||
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
|
||||
|
||||
"postcss": ["postcss@8.5.9", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-7a70Nsot+EMX9fFU3064K/kdHWZqGVY+BADLyXc8Dfv+mTLLVl6JzJpPaCZ2kQL9gIJvKXSLMHhqdRRjwQeFtw=="],
|
||||
"postcss": ["postcss@8.5.25", "", { "dependencies": { "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw=="],
|
||||
|
||||
"react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="],
|
||||
|
||||
@@ -250,8 +254,36 @@
|
||||
|
||||
"update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="],
|
||||
|
||||
"vite": ["vite@6.4.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ=="],
|
||||
"vite": ["vite@6.4.3", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A=="],
|
||||
|
||||
"yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="],
|
||||
|
||||
"@types/babel__core/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="],
|
||||
|
||||
"@types/babel__core/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@types/babel__generator/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@types/babel__template/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="],
|
||||
|
||||
"@types/babel__template/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@types/babel__traverse/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@types/babel__core/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@types/babel__core/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
|
||||
"@types/babel__generator/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@types/babel__generator/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
|
||||
"@types/babel__template/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@types/babel__template/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
|
||||
"@types/babel__traverse/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@types/babel__traverse/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+56
File diff suppressed because one or more lines are too long
Vendored
-56
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-DqP-zmqH.js"></script>
|
||||
<script type="module" crossorigin src="/admin/assets/index-CviJXT-1.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
+5
-1
@@ -15,7 +15,11 @@
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"vite": "^6.3.3",
|
||||
"vite": "^6.4.3",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"overrides": {
|
||||
"@babel/core": "^7.29.6",
|
||||
"postcss": "^8.5.23"
|
||||
}
|
||||
}
|
||||
|
||||
+12
-2
@@ -39,11 +39,21 @@ export const api = {
|
||||
stats: () => apiFetch('/admin/api/stats'),
|
||||
health: () => apiFetch('/admin/api/health-indicators'),
|
||||
agents: () => apiFetch('/admin/api/agents'),
|
||||
sources: () => apiFetch('/admin/api/sources'),
|
||||
requests: (page = 1, qs = '') => apiFetch(`/admin/api/requests?page=${page}${qs}`),
|
||||
apiKeys: () => apiFetch('/admin/api/api-keys'),
|
||||
createApiKey: (name: string) => apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
revokeApiKey: (name: string) => apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
createApiKey(keyName: string) {
|
||||
return apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name: keyName }) });
|
||||
},
|
||||
revokeApiKey(keyName: string) {
|
||||
return apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name: keyName }) });
|
||||
},
|
||||
updateClientTtl: (clientId: string, tokenTtl: number | null) => apiFetch('/admin/api/update-client-ttl', { method: 'POST', body: JSON.stringify({ clientId, tokenTtl }) }),
|
||||
rescopeClient: (clientId: string, sourceId: string, federatedRead: string[]) =>
|
||||
apiFetch('/admin/api/rescope-client', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ clientId, sourceId, federatedRead }),
|
||||
}),
|
||||
revokeClient: (clientId: string) => apiFetch('/admin/api/revoke-client', { method: 'POST', body: JSON.stringify({ clientId }) }),
|
||||
// v0.36.1.0 (T15 / E6) — calibration endpoints.
|
||||
calibrationProfile: (holder?: string) =>
|
||||
|
||||
+169
-4
@@ -18,6 +18,8 @@ interface Agent {
|
||||
client_name?: string; // compat
|
||||
grant_types: string[];
|
||||
scope: string;
|
||||
source_id: string | null;
|
||||
federated_read: string[];
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
total_requests: number;
|
||||
@@ -26,6 +28,12 @@ interface Agent {
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
federated: boolean;
|
||||
}
|
||||
|
||||
interface ApiKey {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -36,6 +44,7 @@ interface ApiKey {
|
||||
|
||||
export function AgentsPage() {
|
||||
const [agents, setAgents] = useState<Agent[]>([]);
|
||||
const [sources, setSources] = useState<Source[]>([]);
|
||||
const [hideRevoked, setHideRevoked] = useState(true);
|
||||
const [showRegister, setShowRegister] = useState(false);
|
||||
const [showCredentials, setShowCredentials] = useState<{ clientId: string; clientSecret: string; name: string } | null>(null);
|
||||
@@ -43,7 +52,10 @@ export function AgentsPage() {
|
||||
const [showApiKeyToken, setShowApiKeyToken] = useState<{ name: string; token: string } | null>(null);
|
||||
const [selectedAgent, setSelectedAgent] = useState<Agent | null>(null);
|
||||
|
||||
useEffect(() => { loadAgents(); }, []);
|
||||
useEffect(() => {
|
||||
loadAgents();
|
||||
api.sources().then(setSources).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const loadAgents = () => { api.agents().then(setAgents).catch(() => {}); };
|
||||
|
||||
@@ -88,6 +100,7 @@ export function AgentsPage() {
|
||||
<th>Name</th>
|
||||
<th>Type</th>
|
||||
<th>Scopes</th>
|
||||
<th>Sources</th>
|
||||
<th>Status</th>
|
||||
<th>Requests</th>
|
||||
<th>Last Used</th>
|
||||
@@ -108,6 +121,11 @@ export function AgentsPage() {
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}
|
||||
</td>
|
||||
<td style={{ color: 'var(--text-secondary)', fontSize: 12 }}>
|
||||
{a.auth_type === 'oauth'
|
||||
? `${a.source_id || 'none'} · ${(a.federated_read || []).length} readable`
|
||||
: 'Unscoped'}
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${a.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{a.status}</span>
|
||||
</td>
|
||||
@@ -144,7 +162,21 @@ export function AgentsPage() {
|
||||
)}
|
||||
|
||||
{selectedAgent && (
|
||||
<AgentDrawer agent={selectedAgent} onClose={() => setSelectedAgent(null)} onRevoked={loadAgents} />
|
||||
<AgentDrawer
|
||||
key={selectedAgent.id}
|
||||
agent={selectedAgent}
|
||||
sources={sources}
|
||||
onClose={() => setSelectedAgent(null)}
|
||||
onRevoked={loadAgents}
|
||||
onRescoped={({ sourceId, federatedRead }) => {
|
||||
setSelectedAgent(current => current ? {
|
||||
...current,
|
||||
source_id: sourceId,
|
||||
federated_read: federatedRead,
|
||||
} : current);
|
||||
loadAgents();
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showApiKeyCreate && (
|
||||
@@ -381,7 +413,127 @@ function CredentialsModal({ credentials, onClose }: {
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
|
||||
function SourceAccessEditor({ clientId, agent, sources, onRescoped }: {
|
||||
clientId: string;
|
||||
agent: Agent;
|
||||
sources: Source[];
|
||||
onRescoped: (scope: { sourceId: string; federatedRead: string[] }) => void;
|
||||
}) {
|
||||
const [writeSource, setWriteSource] = useState(agent.source_id || 'default');
|
||||
const [readSources, setReadSources] = useState<string[]>(agent.federated_read || []);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
const [saved, setSaved] = useState(false);
|
||||
const readableSet = new Set(readSources);
|
||||
const activeSourceIds = new Set(sources.map(source => source.id));
|
||||
const unavailableReadSources = readSources.filter(sourceId => !activeSourceIds.has(sourceId));
|
||||
const primaryUnavailable = !activeSourceIds.has(writeSource);
|
||||
|
||||
const save = async () => {
|
||||
if (readSources.length === 0) {
|
||||
setError('Select at least one readable source.');
|
||||
return;
|
||||
}
|
||||
setSaving(true);
|
||||
setError('');
|
||||
setSaved(false);
|
||||
try {
|
||||
const result = await api.rescopeClient(clientId, writeSource, readSources) as {
|
||||
sourceId: string;
|
||||
federatedRead: string[];
|
||||
};
|
||||
setWriteSource(result.sourceId);
|
||||
setReadSources(result.federatedRead);
|
||||
setSaved(true);
|
||||
onRescoped(result);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : 'Failed to save source access');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="section-title">Source Access</div>
|
||||
<div style={{ color: 'var(--text-secondary)', fontSize: 12, lineHeight: 1.5, marginBottom: 12 }}>
|
||||
The primary source is the write destination. Read access is an explicit allowlist and does not widen automatically.
|
||||
</div>
|
||||
<div style={{ marginBottom: 14 }}>
|
||||
<label htmlFor="agent-write-source">Primary / write source</label>
|
||||
<select
|
||||
id="agent-write-source"
|
||||
value={writeSource}
|
||||
onChange={e => { setWriteSource(e.target.value); setSaved(false); }}
|
||||
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 14 }}
|
||||
>
|
||||
{primaryUnavailable && (
|
||||
<option value={writeSource} disabled>{writeSource} · unavailable</option>
|
||||
)}
|
||||
{sources.map(source => (
|
||||
<option key={source.id} value={source.id}>{source.name} ({source.id})</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<fieldset style={{ border: 0, padding: 0, margin: '0 0 14px' }}>
|
||||
<legend>Readable sources</legend>
|
||||
<div className="checkbox-group" style={{ marginTop: 6 }}>
|
||||
{sources.map(source => (
|
||||
<label key={source.id} className="checkbox-label">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={readableSet.has(source.id)}
|
||||
onChange={e => {
|
||||
setSaved(false);
|
||||
setReadSources(current => e.target.checked
|
||||
? [...current, source.id]
|
||||
: current.filter(id => id !== source.id));
|
||||
}}
|
||||
/>
|
||||
{source.name} ({source.id}){source.federated ? ' · federated' : ' · private'}
|
||||
</label>
|
||||
))}
|
||||
{unavailableReadSources.map(sourceId => (
|
||||
<label key={sourceId} className="checkbox-label" style={{ color: 'var(--warning)' }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked
|
||||
onChange={() => {
|
||||
setSaved(false);
|
||||
setReadSources(current => current.filter(id => id !== sourceId));
|
||||
}}
|
||||
/>
|
||||
{sourceId} · unavailable (clear to remove grant)
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
{(primaryUnavailable || unavailableReadSources.length > 0) && (
|
||||
<div style={{ color: 'var(--warning)', fontSize: 13, marginBottom: 10 }}>
|
||||
This client references unavailable or archived sources. Choose an active primary source and clear unavailable read grants before saving.
|
||||
</div>
|
||||
)}
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 10 }}>{error}</div>}
|
||||
{saved && <div style={{ color: 'var(--success)', fontSize: 13, marginBottom: 10 }}>Source access saved.</div>}
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={saving || readSources.length === 0 || sources.length === 0 || primaryUnavailable || unavailableReadSources.length > 0}
|
||||
onClick={save}
|
||||
>
|
||||
{saving ? 'Saving...' : 'Save Source Access'}
|
||||
</button>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, sources, onClose, onRevoked, onRescoped }: {
|
||||
agent: Agent;
|
||||
sources: Source[];
|
||||
onClose: () => void;
|
||||
onRevoked: () => void;
|
||||
onRescoped: (scope: { sourceId: string; federatedRead: string[] }) => void;
|
||||
}) {
|
||||
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const serverUrl = window.location.origin;
|
||||
@@ -553,6 +705,15 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
|
||||
</div>
|
||||
|
||||
{isOAuth && (
|
||||
<SourceAccessEditor
|
||||
clientId={cid}
|
||||
agent={agent}
|
||||
sources={sources}
|
||||
onRescoped={onRescoped}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/*
|
||||
Config Export visible for both auth_type=oauth AND auth_type=api_key.
|
||||
Claude Code + Cursor + JSON tabs render real snippets regardless
|
||||
@@ -579,7 +740,11 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
{(() => {
|
||||
const oauthOnlyTabs = new Set(['chatgpt', 'claude-cowork', 'perplexity']);
|
||||
if (!isOAuth && oauthOnlyTabs.has(tab)) {
|
||||
const clientName = { chatgpt: 'ChatGPT', 'claude-cowork': 'Claude.ai', perplexity: 'Perplexity' }[tab] || tab;
|
||||
const clientName = tab === 'chatgpt'
|
||||
? 'ChatGPT'
|
||||
: tab === 'claude-cowork'
|
||||
? 'Claude.ai'
|
||||
: 'Perplexity';
|
||||
return (
|
||||
<div style={{
|
||||
background: 'rgba(255, 200, 100, 0.08)',
|
||||
|
||||
@@ -21,7 +21,7 @@ export function DashboardPage() {
|
||||
api.stats().then(setStats).catch(() => {});
|
||||
api.health().then(setHealth).catch(() => {});
|
||||
|
||||
const es = new EventSource('/admin/events');
|
||||
const es = new EventSource('/admin/events', { withCredentials: true });
|
||||
eventSourceRef.current = es;
|
||||
es.onopen = () => setSseStatus('connected');
|
||||
es.onmessage = (e) => {
|
||||
|
||||
@@ -26,8 +26,8 @@
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"heic-decode": "^2.1.0",
|
||||
"js-yaml": "^3.14.2",
|
||||
"marked": "^18.0.0",
|
||||
"js-yaml": "^3.15.0",
|
||||
"marked": "^18.0.2",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
@@ -50,6 +50,18 @@
|
||||
"trustedDependencies": [
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"body-parser": "^2.3.0",
|
||||
"fast-uri": "^3.1.5",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
"hono": "^4.12.34",
|
||||
"ip-address": "^10.3.1",
|
||||
"js-yaml": "^3.15.0",
|
||||
"qs": "^6.15.2",
|
||||
},
|
||||
"packages": {
|
||||
"@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.74", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Xew9rfz9WWhDSyF8rNhjT/XWOWelNfJrMlmG0Ahw210hStisRpQZ1s+7VeI9JTJOZ5y5tXqBi5kfPwYnCfyRTA=="],
|
||||
|
||||
@@ -151,7 +163,7 @@
|
||||
|
||||
"@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="],
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
|
||||
"@hono/node-server": ["@hono/node-server@2.0.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA=="],
|
||||
|
||||
"@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="],
|
||||
|
||||
@@ -159,6 +171,8 @@
|
||||
|
||||
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
|
||||
|
||||
"@nodable/entities": ["@nodable/entities@3.0.0", "", {}, "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw=="],
|
||||
|
||||
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
|
||||
|
||||
"@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="],
|
||||
@@ -307,11 +321,13 @@
|
||||
|
||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
||||
|
||||
"anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="],
|
||||
|
||||
"argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
|
||||
"body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="],
|
||||
|
||||
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
|
||||
|
||||
@@ -385,15 +401,15 @@
|
||||
|
||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
|
||||
"fast-uri": ["fast-uri@3.1.5", "", {}, "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw=="],
|
||||
|
||||
"fast-xml-builder": ["fast-xml-builder@1.1.4", "", { "dependencies": { "path-expression-matcher": "^1.1.3" } }, "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg=="],
|
||||
"fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="],
|
||||
|
||||
"fast-xml-parser": ["fast-xml-parser@5.5.8", "", { "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.2.0", "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ=="],
|
||||
"fast-xml-parser": ["fast-xml-parser@5.10.1", "", { "dependencies": { "@nodable/entities": "^3.0.0", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.1", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw=="],
|
||||
|
||||
"finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
|
||||
|
||||
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
|
||||
"form-data": ["form-data@4.0.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ=="],
|
||||
|
||||
"form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
|
||||
|
||||
@@ -417,11 +433,11 @@
|
||||
|
||||
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
"hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="],
|
||||
|
||||
"heic-decode": ["heic-decode@2.1.0", "", { "dependencies": { "libheif-js": "^1.19.8" } }, "sha512-0fB3O3WMk38+PScbHLVp66jcNhsZ/ErtQ6u2lMYu/YxXgbBtl+oKOhGQHa4RpvE68k8IzbWkABzHnyAIjR758A=="],
|
||||
|
||||
"hono": ["hono@4.12.10", "", {}, "sha512-mx/p18PLy5og9ufies2GOSUqep98Td9q4i/EF6X7yJgAiIopxqdfIO3jbqsi3jRgTgw88jMDEzVKi+V2EF+27w=="],
|
||||
"hono": ["hono@4.13.0", "", {}, "sha512-jhunvfHWxd7J5EFfSgH4xsYJzSe/lfqbUCxiyyeaQasUsXeEHXtzVid+7EOGByc5JnFa23SSFL3Y2RV/z1T+eQ=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
|
||||
|
||||
@@ -431,7 +447,7 @@
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"ip-address": ["ip-address@10.1.0", "", {}, "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q=="],
|
||||
"ip-address": ["ip-address@10.4.0", "", {}, "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ=="],
|
||||
|
||||
"ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
|
||||
|
||||
@@ -439,11 +455,13 @@
|
||||
|
||||
"is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
|
||||
|
||||
"is-unsafe": ["is-unsafe@2.0.0", "", {}, "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA=="],
|
||||
|
||||
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
|
||||
|
||||
"jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="],
|
||||
|
||||
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
"js-yaml": ["js-yaml@3.15.0", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog=="],
|
||||
|
||||
"json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],
|
||||
|
||||
@@ -455,7 +473,7 @@
|
||||
|
||||
"libheif-js": ["libheif-js@1.19.8", "", {}, "sha512-vQJWusIxO7wavpON1dusciL8Go9jsIQ+EUrckauFYAiSTjcmLAsuJh3SszLpvkwPci3JcL41ek2n+LUZGFpPIQ=="],
|
||||
|
||||
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
||||
"marked": ["marked@18.0.6", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-MrV5puXBfuiy6wl6DLaq3BtIJQAJToAd5zt/ZKhRfGRAuFPALE7/4Y7jnxRQoEgK/pBgurGqLyAuRgZ2xOjr6w=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
@@ -487,7 +505,7 @@
|
||||
|
||||
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
|
||||
|
||||
"path-expression-matcher": ["path-expression-matcher@1.5.0", "", {}, "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ=="],
|
||||
"path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="],
|
||||
|
||||
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
||||
|
||||
@@ -503,7 +521,7 @@
|
||||
|
||||
"pure-rand": ["pure-rand@8.4.0", "", {}, "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A=="],
|
||||
|
||||
"qs": ["qs@6.15.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ=="],
|
||||
"qs": ["qs@6.15.3", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A=="],
|
||||
|
||||
"range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
|
||||
|
||||
@@ -529,9 +547,9 @@
|
||||
|
||||
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||
|
||||
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
|
||||
"side-channel": ["side-channel@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4", "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ=="],
|
||||
|
||||
"side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="],
|
||||
"side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="],
|
||||
|
||||
"side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="],
|
||||
|
||||
@@ -543,7 +561,7 @@
|
||||
|
||||
"strip-bom-string": ["strip-bom-string@1.0.0", "", {}, "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g=="],
|
||||
|
||||
"strnum": ["strnum@2.2.3", "", {}, "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg=="],
|
||||
"strnum": ["strnum@2.4.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg=="],
|
||||
|
||||
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
|
||||
|
||||
@@ -577,6 +595,8 @@
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
|
||||
"xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="],
|
||||
|
||||
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||
|
||||
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
|
||||
@@ -595,12 +615,20 @@
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="],
|
||||
|
||||
"body-parser/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="],
|
||||
|
||||
"es-set-tostringtag/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"get-intrinsic/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@anthropic-ai/sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
|
||||
+6
-1
@@ -13,4 +13,9 @@ timeout = 60_000
|
||||
# fixtures still match the schema. v0.37's production default is ZE/1280;
|
||||
# tests that want the new default call configureGateway() explicitly in
|
||||
# their own beforeAll.
|
||||
preload = ["./test/helpers/legacy-embedding-preload.ts"]
|
||||
#
|
||||
# #2823: redirect GBRAIN_AUDIT_DIR to a per-run scratch dir BEFORE any test
|
||||
# runs, so audit-emitting code paths (content-sanity, shell-audit, etc.)
|
||||
# can't leak fixture events into the operator's real ~/.gbrain/audit/. See
|
||||
# test/helpers/audit-dir-preload.ts for the full rationale.
|
||||
preload = ["./test/helpers/legacy-embedding-preload.ts", "./test/helpers/audit-dir-preload.ts"]
|
||||
|
||||
@@ -148,6 +148,51 @@ RRF fusion, multi-query expansion, and 4-layer dedup are engine-agnostic. They o
|
||||
|
||||
**Why not self-hosted for v0:** The brain should be infrastructure agents use, not something you maintain. Self-hosted Postgres with Docker is a welcome community PR, but v0 optimizes for zero ops.
|
||||
|
||||
### Opt-in RLS source-scope binding (`GBRAIN_RLS_SCOPE_BINDING`)
|
||||
|
||||
Defense-in-depth layer for Postgres deployments that want the database itself
|
||||
to enforce source isolation, in addition to the mandatory app-layer filters
|
||||
(`sourceScopeOpts` — layer 1, always on).
|
||||
|
||||
**Mechanism.** With `GBRAIN_RLS_SCOPE_BINDING=1` (or `true`), the engine's
|
||||
source-scoped read methods wrap their queries in a transaction that first runs
|
||||
`SELECT set_config('app.scopes', $1, true)` — the value is a bound parameter
|
||||
(federated `sourceIds` CSV > scalar `sourceId` > `'*'` for unscoped internal
|
||||
reads), transaction-local (equivalent to `SET LOCAL`, which itself can't take
|
||||
bound params). An RLS policy can then filter rows by
|
||||
`current_setting('app.scopes', true)`.
|
||||
|
||||
**Default off.** With the env var unset, reads call through on the shared pool
|
||||
exactly as before — no per-read transaction, no pool-slot hold (the search
|
||||
methods keep the transaction they always had for their `SET LOCAL
|
||||
statement_timeout`). Existing operators see zero behavior change.
|
||||
|
||||
**Enabling it** (operator-managed SQL; gbrain ships no DDL for this):
|
||||
|
||||
```sql
|
||||
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
|
||||
CREATE POLICY pages_scope_filter ON pages
|
||||
USING (current_setting('app.scopes', true) = '*'
|
||||
OR source_id = ANY(string_to_array(current_setting('app.scopes', true), ',')));
|
||||
|
||||
-- Required: connections that don't run through the scoped read helper
|
||||
-- (admin, autopilot, cycle, writes) must default to unscoped, or they
|
||||
-- see zero rows once the policy exists:
|
||||
ALTER ROLE <runtime-role> SET app.scopes = '*';
|
||||
|
||||
-- If the runtime role OWNS the table, RLS is skipped for it unless forced:
|
||||
ALTER TABLE pages FORCE ROW LEVEL SECURITY;
|
||||
```
|
||||
|
||||
Safe to enable in either order: the env var without a policy is a no-op
|
||||
setting; a policy without the env var is enforced only via the role default.
|
||||
|
||||
**Honest caveat:** only read paths routed through the scoped helper carry a
|
||||
per-request scope binding — unwrapped paths (writes, admin/maintenance reads)
|
||||
run under the role default and are not backstopped per caller. This is layer 2;
|
||||
the app-layer source filters remain layer 1 and stay mandatory. Behavioral pins
|
||||
live in `test/postgres-engine-rls-scope.test.ts`.
|
||||
|
||||
## PGLiteEngine (v0.7, ships)
|
||||
|
||||
**Dependencies:** `@electric-sql/pglite` (v0.4.4+)
|
||||
|
||||
+37
-1
@@ -39,10 +39,11 @@ gbrain migrate --to pglite # Postgres → PGLite (rare)
|
||||
|
||||
For shared / large / multi-machine deployments (a team or company brain with multiple users hitting one server over HTTP MCP with OAuth scoping per user), follow the dedicated walkthrough: **[Tutorial: set up GBrain as your company brain](tutorials/company-brain.md)**.
|
||||
|
||||
API keys live in `~/.gbrain/config.json` (file plane) or env vars (`OPENAI_API_KEY`, `ZEROENTROPY_API_KEY`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY`). Set via CLI:
|
||||
API keys live in `~/.gbrain/config.json` (file plane) or env vars (`OPENAI_API_KEY`, `OPENROUTER_API_KEY`, `ZEROENTROPY_API_KEY`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY`). Set via CLI:
|
||||
|
||||
```bash
|
||||
gbrain config set zeroentropy_api_key sk-...
|
||||
gbrain config set openrouter_api_key sk-or-...
|
||||
gbrain config set anthropic_api_key sk-ant-...
|
||||
```
|
||||
|
||||
@@ -112,3 +113,38 @@ gbrain models doctor # 1-token probe per configured model
|
||||
```
|
||||
|
||||
If anything's yellow, `gbrain doctor` names the fix command in the message. Most issues are missing API keys or stale schema (`gbrain upgrade --force-schema`).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### PGLite crashes on macOS 26.x (Tahoe)
|
||||
|
||||
PGLite's embedded WASM engine is incompatible with macOS 26.x (Tahoe) on Apple Silicon. If `gbrain init --pglite` crashes during engine initialization, switch to native Homebrew PostgreSQL:
|
||||
|
||||
```bash
|
||||
# Install PostgreSQL + pgvector
|
||||
brew install postgresql@17
|
||||
brew services start postgresql@17
|
||||
createdb gbrain
|
||||
|
||||
# Build pgvector from source (required for vector search)
|
||||
cd /tmp && git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector && make && make install
|
||||
psql gbrain -c "CREATE EXTENSION IF NOT EXISTS vector;"
|
||||
|
||||
# Point gbrain at your local Postgres
|
||||
cat > ~/.gbrain/config.json << 'EOF'
|
||||
{
|
||||
"engine": "postgres",
|
||||
"database_url": "postgresql://localhost:5432/gbrain",
|
||||
"schema_pack": "gbrain-base-v2"
|
||||
}
|
||||
EOF
|
||||
|
||||
# Run migrations and verify
|
||||
gbrain apply-migrations --yes
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
All 102 migrations run on first try. Once `gbrain doctor` shows green, the brain works identically to PGLite — same commands, same skills, same data model. The only difference is the storage backend.
|
||||
|
||||
> **Note:** This workaround is temporary. When the upstream WASM runtime fix ships (likely via a Bun update), `--pglite` will work on Tahoe again.
|
||||
|
||||
@@ -362,6 +362,39 @@ done
|
||||
|
||||
If any SHA differs from what's in the workflow files, update the pin and version comment.
|
||||
|
||||
## GitHub releases (binary assets + self-update) — #3521
|
||||
|
||||
`.github/workflows/release.yml` publishes a GitHub release automatically for
|
||||
**every VERSION bump that lands on master** (trigger: push to master touching
|
||||
`VERSION`, plus `workflow_dispatch` for a manual first run or repair). No
|
||||
manual tag push is part of the ship flow — the workflow reads `VERSION` (the
|
||||
single source of truth), mints tag `v<VERSION>` at the pushed commit, titles
|
||||
the release the same, uses that version's `CHANGELOG.md` entry as the notes
|
||||
(`scripts/changelog-entry.sh`; falls back to a CHANGELOG link if the entry is
|
||||
missing), and attaches the compiled binaries.
|
||||
|
||||
Why every bump, not selective: `gbrain check-update` resolves the latest
|
||||
version from `VERSION` on master, while binary self-update
|
||||
(`src/core/binary-self-update.ts`) downloads assets from `releases/latest`.
|
||||
Any release that lags `VERSION` tells binary installs an upgrade exists that
|
||||
self-update cannot apply. `releases/latest` must track `VERSION`.
|
||||
|
||||
Invariants:
|
||||
|
||||
- **Asset names are a contract.** The build matrix's `artifact:` names must
|
||||
equal what `expectedAssetName()` in `src/core/binary-self-update.ts`
|
||||
returns (`gbrain-darwin-arm64`, `gbrain-linux-x64` today). Adding a
|
||||
platform means updating BOTH plus the version job's completeness check;
|
||||
`test/release-workflow.test.ts` pins all of it.
|
||||
- **Idempotent + self-repairing.** The version job skips when a release for
|
||||
`v<VERSION>` already exists with all expected assets; a partial release
|
||||
(tag but no release, or missing assets) is completed on re-run. Racing
|
||||
master pushes queue via the `release` concurrency group — a skipped
|
||||
intermediate version is fine, latest is what matters.
|
||||
- **Historical tags are never rewritten.** Old 3-segment versions keep their
|
||||
history; every new 4-segment `VERSION` mints a fresh tag.
|
||||
- **Permissions stay scoped.** `contents: write` lives on the release job
|
||||
only; everything else runs read-only.
|
||||
|
||||
## PR descriptions cover the whole branch
|
||||
|
||||
|
||||
+43
-1
@@ -3,6 +3,8 @@
|
||||
On-demand reference (see CLAUDE.md Reference map). Current behavior + invariants
|
||||
only.
|
||||
|
||||
`test/e2e/serve-http-oauth.test.ts` additionally pins confidential POST/Basic revocation, public-client SDK fallthrough, malformed/mixed authentication rejection, cross-client isolation, unknown-token opacity, metadata auth methods, no-store responses, strict post-revoke `401`, and retryable backend `503` semantics.
|
||||
|
||||
### Test command tiers
|
||||
|
||||
Seven test command tiers, each with a clear scope:
|
||||
@@ -17,6 +19,32 @@ Seven test command tiers, each with a clear scope:
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential. | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | The historical pre-check scripts (22, chained sequentially in package.json). Overlaps `verify` heavily but is NOT a superset — `verify`'s `CHECKS` array in `scripts/run-verify-parallel.sh` (~30 entries incl. typecheck) is the authoritative gate; `check:all` keeps a few local-only extras (trailing-newline, exports-count, no-legacy-getconnection). | ~10s | Local-only sweep for the extras. |
|
||||
|
||||
### Shell dispatch and Windows
|
||||
|
||||
All four of `test`, `verify`, `ci:local` and `test:e2e` hand off to shell scripts
|
||||
under `scripts/`, so every `check:*` entry in `package.json` invokes its script as
|
||||
`bash scripts/<name>.sh` instead of relying on the shebang — bun on Windows cannot
|
||||
exec a `.sh` directly. Add a new shell-script check with that same prefix. The
|
||||
`scripts/*.ts` entries run under bun and take no prefix.
|
||||
|
||||
The scripts must also be on disk with Unix line endings. A strict bash (WSL, Linux
|
||||
CI, macOS) rejects CRLF and dies on the script's first meaningful line; the Cygwin
|
||||
bash that ships with Git for Windows tolerates it, so a green local run is not by
|
||||
itself evidence that a script is CRLF-clean.
|
||||
The root `.gitattributes` pins `*.sh text eol=lf`, which overrides the
|
||||
`core.autocrlf=true` default that Git for Windows installs. It pins `*.md` the
|
||||
same way, because the frontmatter readers anchor on a `---` fence followed by a
|
||||
Unix line ending and a CRLF checkout makes a document parse as having no
|
||||
frontmatter, silently. Working copies cloned
|
||||
before those pins need a one-time `git rm --cached -r . -q && git reset --hard` to
|
||||
pick them up; see the Windows section of `CONTRIBUTING.md`.
|
||||
|
||||
Wallclock figures in the table above are from a Mac dev box. Windows is
|
||||
substantially slower because each check pays full process-creation cost, and three
|
||||
tree-walking checks (`check:privacy`, `check:test-names`, `check:test-isolation`)
|
||||
plus `typecheck` can exceed the 120s per-check cap in `run-verify-parallel.sh`
|
||||
there even though they pass on Linux and macOS.
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` across 10 matrix shards partitioned by weight-aware LPT bin-packing (`scripts/sharding.ts`) and INCLUDES `*.slow.test.ts` (the two outlier slow files run as dedicated jobs alongside the matrix). CI EXCLUDES `*.serial.test.ts` from the shards and runs them in a dedicated job via `bun run test:serial`, one bun process per file — keeping serial files out of the shard processes is what preserves the `mock.module` quarantine (a top-level mock in one file leaks into every other file sharing its process). `bun run verify` gets its own job too. CI is the ground truth for "did everything pass."
|
||||
@@ -44,6 +72,15 @@ If a shard wedges (per-shard `GBRAIN_TEST_SHARD_TIMEOUT` cap, default 600s), the
|
||||
- `tests/heavy/*.sh` → ops-shape shell scripts. Cost minutes per run; NOT in default `bun test`. Run via `bun run test:heavy` or scheduled nightly via `.github/workflows/heavy-tests.yml`. Examples: pg_upgrade matrix (boot legacy brain → walk to head), RSS budget gate (measure peak worker RSS vs committed baseline), read-latency-under-sync (p50/p95/p99 under concurrent writer load), sync lock regression (N concurrent syncs assert 1 winner + N-1 lock-busy + zero leaked `gbrain_cycle_locks` rows). See `tests/heavy/README.md` for when to add a script here vs `*.slow.test.ts`. Files prefixed with `_` (e.g. `tests/heavy/_build_legacy_fixtures.sh`) are helpers/libs invoked by sibling tests — the runner skips them.
|
||||
- `test/fuzz/*.test.ts` → property-based fuzz harness. Pure-validator targets in `pure-validators.test.ts` are guarded by `scripts/check-fuzz-purity.sh` (in `bun run verify`), which `bun build --target=bun` bundles each target and greps the resulting bundle for banned transitive imports (`node:fs`, `node:child_process`, engine modules). Anything that fails the guard moves to `mixed-validators.test.ts` (still property-tested, but no purity guarantee) or `filesystem-validators.test.ts` (fs-backed, uses temp dirs). Fuzz tests run in the default `bun test` loop because they're fast (~3s for ~12 properties × 1000 runs each).
|
||||
|
||||
### Skills-manifest freshness guard
|
||||
|
||||
`skills/skills.lock.json` is a committed sha256 inventory of every bundled file under
|
||||
`skills/` (tamper evidence, not signatures — see `src/core/skills-integrity.ts`).
|
||||
Any change under `skills/` must regenerate it: `bun run scripts/generate-skills-manifest.ts`.
|
||||
`scripts/check-skills-manifest-fresh.sh` (`bun run check:skills-manifest`, wired into
|
||||
`bun run verify`) regenerates to a tmp file and diffs, failing CI on drift; at runtime
|
||||
`gbrain doctor` reports the same drift as a warn-only `skills_manifest_integrity` check.
|
||||
|
||||
### Test-isolation lint and helpers
|
||||
|
||||
The cross-file flake class is enforced statically by `scripts/check-test-isolation.sh`, wired into `bun run verify` and `bun run check:all`. Rules (non-serial unit files only; `*.serial.test.ts` and `test/e2e/*` are skipped):
|
||||
@@ -187,8 +224,10 @@ Unit tests and what they cover:
|
||||
- `test/orphans.test.ts` — orphans command: detection, pseudo filtering, text/json/count outputs, MCP op.
|
||||
- `test/postgres-engine.test.ts` — `statement_timeout` scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against a reintroduced bare `SET statement_timeout`.
|
||||
- `test/sync.test.ts` — sync logic + regression guard asserting top-level `engine.transaction` is not called.
|
||||
- `test/sync-pull-failed-anchor.serial.test.ts` — #3068 regression: a failed internal `git pull` (local-path origin vs `protocol.file.allow=never`) with zero imports returns `partial`/`pull_failed` (not `up_to_date`), freezes `last_commit` + `last_sync_at`, recovers after a manual pull; fall-through import of local commits preserved. Serial: pins `GBRAIN_HOME` to a temp dir for the whole file.
|
||||
- `test/sync-concurrency.test.ts` — `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping; `shouldRunParallel()` explicit-bypasses-floor contract; `parseWorkers()` validation rejecting `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars.
|
||||
- `test/sync-parallel.test.ts` — PGLite-routed coverage of the bookmark gate under concurrency, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract.
|
||||
- `test/sync-all-missing-path.test.ts` — `sync --all --missing-path <fail|skip>` pure helpers: `parseMissingPathMode` (default fail, explicit values, loud rejection of bad/dangling values, never swallows a following flag) and `partitionMissingPathSources` (classification driven only by the injected pathExists predicate — no fs; null `local_path` passes through runnable; order preserved).
|
||||
- `test/sync-failures.test.ts` — `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts` and `import-file.ts`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` `AcknowledgeResult` shape + backfill on legacy entries.
|
||||
- `test/doctor.test.ts` — doctor command; assertions that `jsonb_integrity` scans the four JSONB write sites and `markdown_body_completeness` is present.
|
||||
- `test/utils.test.ts` — shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics.
|
||||
@@ -239,8 +278,11 @@ E2E tests live in `test/e2e/` and run against real Postgres+pgvector (require `D
|
||||
- `test/e2e/http-transport.test.ts` — `gbrain serve --http` end-to-end against real Postgres: bearer auth round-trip, `last_used_at` SQL-level debounce, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the dispatch round-trip with a real operation. Skips without `DATABASE_URL`.
|
||||
- `test/e2e/serve-http-oauth.test.ts` — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. Real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire, RFC 7591 §3.2.1); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance contract:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }`. Reference fix for the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Also covers the trust-boundary fix: an HTTP MCP `submit_job` for `name: "shell"` MUST reject with a permission error (request handler sets `remote: true` and `submit_job`'s protected-name guard fires), and the same guard rejects subagent submission. Skips without `DATABASE_URL`.
|
||||
- `test/e2e/sync-parallel.test.ts` — `DATABASE_URL`-gated. 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx`. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- `test/e2e/multi-source-bug-class.test.ts` — PGLite in-memory regression suite pinning every multi-source bug site: `listAllPageRefs` ordering by `(source_id, slug)`, `getPage` with sourceId picks the right `(source, slug)` row, `extract-takes` processes both overlapping `people/alice` rows independently, `listPages` filters correctly with `PageFilters.sourceId`, `addLinksBatch` with `from/to_source_id` targets the right rows, `validateSourceId` rejects path traversal, reverse-write disk layout uses `brainDir/.sources/<id>/<slug>.md` for non-default sources. No `DATABASE_URL` needed. Wired into `scripts/e2e-test-map.ts` so changes to extract-takes / patterns / synthesize / embed / extract / migrate-engine auto-trigger it.
|
||||
- `test/e2e/multi-source-bug-class.test.ts` — PGLite in-memory regression suite pinning every multi-source bug site: `listAllPageRefs` ordering by `(source_id, slug)`, `getPage` with sourceId picks the right `(source, slug)` row, `extract-takes` processes both overlapping `people/alice` rows independently, `listPages` filters correctly with `PageFilters.sourceId`, `addLinksBatch` with `from/to_source_id` targets the right rows, `validateSourceId` rejects path traversal, reverse-write disk layout uses `brainDir/.sources/<id>/<slug>.md` for non-default sources, `copyMigrationSources` lands source metadata before overlapping-slug pages. No `DATABASE_URL` needed. Wired into `scripts/e2e-test-map.ts` so changes to extract-takes / patterns / synthesize / embed / extract / migrate-engine auto-trigger it.
|
||||
- `test/e2e/migrate-engine-sources-postgres.test.ts` — `DATABASE_URL`-gated companion for `gbrain migrate --to`: migrates a PGLite brain carrying two non-default sources with overlapping slugs into real Postgres and asserts `copyMigrationSources` created every `sources` FK parent (config JSONB intact, not double-encoded) before any page write. Unit-level manifest identity (crash manifest resumes only against the SAME target; legacy engine-only manifests start fresh) is `test/migrate-engine-resume.test.ts`.
|
||||
- `test/e2e/facts-fence-reconcile-postgres.test.ts` — `DATABASE_URL`-gated round-trip for the escape-aware fence parser: renders a `## Facts` fence whose cells carry literal pipes, backslashes (Windows paths), and empty cells via `renderFactsTable`, runs the wipe-and-reinsert reconcile (`runExtractFacts`) on real Postgres, and asserts every cell survives byte-identically with no column shift.
|
||||
- `test/e2e/source-isolation-pglite.test.ts` — PGLite in-memory regression suite pinning the source-isolation seal at two layers. Engine layer: `searchKeyword` / `searchVector` / `searchKeywordChunks` / `listPages` / `getPage` / `traverseGraph` / `traversePaths` apply `sourceId` (scalar fast path) and `sourceIds` (array path) correctly across both engines. Op-handler layer: routes through `sourceScopeOpts(ctx)` so a `read+write`-scoped OAuth client bound to `--source dept-x` cannot see rows from neighboring sources via `search`, `query`, `list_pages`, `get_page`, or `find_experts`. Covers both `ctx.sourceId` (single-source clients) and `ctx.auth.allowedSources` (federated_read clients) precedence; federated array wins over scalar wins over nothing. No `DATABASE_URL` needed.
|
||||
- `test/e2e/think-source-isolation-pglite.test.ts` — PGLite in-memory suite pinning the `think` gather stage's source scope: seeds three sources with cross-source links and embedded takes, then asserts `runGather` under a federated `sourceIds` grant (and under a scalar `sourceId`) keeps every stream — hybrid retrieval, takes keyword + vector (`searchTakes`/`searchTakesVector`), and the `traversePaths` graph walk — inside the grant while still reaching authorized neighboring sources. No `DATABASE_URL` needed.
|
||||
- `test/e2e/skill-brain-first.test.ts` — doctor reports `skill_brain_first` check with structured issues; `--fix --dry-run` previews insertion without writing; `--fix` applies the canonical Convention callout idempotently; `brain_first: exempt` frontmatter resolves the warn; `brain_first_typo` surfaces a paste-ready hint; audit JSONL records `detected` / `resolved` / `fixed` transitions; stable brain emits 0 audit lines/run.
|
||||
- Tier 2 (`test/e2e/skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI.
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees: `find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -87,6 +87,15 @@ embedding proximity. Four layers, added after the incident in
|
||||
deciding "is this page already here, safe to NOT write a duplicate?" keys off
|
||||
`create_safety`, not a raw blended score.
|
||||
|
||||
**Extraction quarantine lane (issue #160):** pages carrying the unverified
|
||||
auto-extracted markers (frontmatter `provenance: auto-extracted` +
|
||||
`status: unverified`, see `src/core/extraction-review.ts`) rank as ordinary
|
||||
content — they are skipped by the compiled-truth fusion boost and by the
|
||||
`people/`/`companies/` namespace source-boost, and every search result from
|
||||
such a page carries `unverified: true` so agents can label the provenance.
|
||||
Promote or reject them via `gbrain extraction-pending` / `gbrain
|
||||
extraction-review`.
|
||||
|
||||
The `search` MCP/CLI op is **cheap-hybrid** (vector + keyword + RRF + pool +
|
||||
title + alias, expansion off); `query` is the full-control variant. NamedThingBench
|
||||
(`gbrain eval retrieval-quality`) gates these families on every PR. Diagnose a
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
# Conversation parser patterns
|
||||
|
||||
The conversation parser turns exported chat and meeting transcripts into a
|
||||
common message stream without requiring an LLM call for known formats. This
|
||||
document describes the built-in pattern contract and the checks required when
|
||||
adding or changing a format.
|
||||
|
||||
## Data flow
|
||||
|
||||
`parseConversation` uses this sequence:
|
||||
|
||||
1. Resolve the page date and timezone context.
|
||||
2. Score every enabled built-in and user pattern against the first ten
|
||||
non-blank lines.
|
||||
3. Re-score the full body when the head score is inconclusive, or when a broad
|
||||
pattern explicitly requires full-body scoring.
|
||||
4. Reject the winner when its acceptance score is below the false-positive
|
||||
floor.
|
||||
5. Apply the winning pattern to every line and attach continuation lines to the
|
||||
preceding message.
|
||||
6. Optionally run LLM polish or fallback when those features are enabled.
|
||||
|
||||
Pattern order is only a tie-breaker. A new regex must be structurally distinct
|
||||
from neighboring formats; moving it earlier in the registry is not a valid
|
||||
non-shadowing strategy.
|
||||
|
||||
## Built-in pattern contract
|
||||
|
||||
Every `PatternEntry` in `builtins.ts` declares:
|
||||
|
||||
- A stable, kebab-case `id`.
|
||||
- A hand-vetted line regex and explicit capture-group indexes.
|
||||
- Where the date comes from and how the time is represented.
|
||||
- A timezone policy.
|
||||
- Whether the format supports multi-line message bodies.
|
||||
- Positive and negative samples that run during module initialization.
|
||||
- A documentation pointer describing the source format.
|
||||
|
||||
The registry refuses to load when a positive sample stops matching, a negative
|
||||
sample starts matching, or a capture map becomes invalid. This catches local
|
||||
regex mistakes before extraction can silently produce empty conversations.
|
||||
|
||||
### Date and timezone rules
|
||||
|
||||
Formats with an inline date should capture it from each message. Time-only
|
||||
formats use an explicit caller fallback first, then the page frontmatter date,
|
||||
then the page effective date. If none is available, the parser uses
|
||||
`1970-01-01` so the missing date remains visible instead of inventing a current
|
||||
date.
|
||||
|
||||
Time-only formats normally use `utc_assumed_with_warn`. The parser constructs a
|
||||
UTC timestamp and returns a timezone warning when the page does not provide a
|
||||
timezone. A new pattern should not imply local-time precision that the source
|
||||
format does not contain.
|
||||
|
||||
### Multi-line messages
|
||||
|
||||
An anchor regex identifies the first line of a message. Subsequent non-anchor
|
||||
lines are appended to that message until another anchor appears. Set
|
||||
`multi_line: true` when continuation content is part of the documented format,
|
||||
such as Markdown bullets, blockquotes, or an exported message body on the next
|
||||
line.
|
||||
|
||||
Tests for a multi-line format should assert the complete message text, including
|
||||
newlines. A message-count assertion alone will not detect lost bullets or a
|
||||
continuation attached to the wrong speaker.
|
||||
|
||||
### Scoring and false positives
|
||||
|
||||
The score compares matched anchors with the pattern's relevant candidate lines.
|
||||
The first pass uses the head of the page for speed. Low-confidence pages are
|
||||
re-scored across the full body before the parser accepts a winner.
|
||||
|
||||
Multi-line formats may opt into `score_continuations_as_body` when their anchor
|
||||
grammar is distinctive. Candidate-only scoring activates only after two anchors
|
||||
match, or when the first non-blank line is an anchor. This evidence threshold
|
||||
lets a single long message keep its continuation body without turning one stray
|
||||
anchor in a prose page into a conversation. Candidate anchor lines that fail the
|
||||
full regex still lower the score. Other patterns continue to use all non-blank
|
||||
lines in their density score.
|
||||
|
||||
Use `score_full_body: true` for a broad grammar that also occurs in ordinary
|
||||
prose. For example, `**Label:** text` can be either a transcript line or a bold
|
||||
label in meeting notes. Narrow formats with a timestamp and a distinctive
|
||||
separator generally do not need this override.
|
||||
|
||||
`quick_reject` is a performance hint, not an acceptance rule. It should cheaply
|
||||
exclude obviously unrelated lines while admitting every string accepted by the
|
||||
main regex.
|
||||
|
||||
## Normalized Slack Markdown
|
||||
|
||||
The `bold-time-dash` pattern parses message anchors shaped like:
|
||||
|
||||
```text
|
||||
**Alice Example** 09:15 — first message
|
||||
- supporting detail
|
||||
**Bob Example** 09:18 — second message
|
||||
```
|
||||
|
||||
Its grammar is:
|
||||
|
||||
```text
|
||||
**speaker** H:MM <dash> text
|
||||
```
|
||||
|
||||
where:
|
||||
|
||||
- `H:MM` is a valid 24-hour time from `0:00` through `23:59`.
|
||||
- `<dash>` may be an em dash (`—`), en dash (`–`), or ASCII hyphen (`-`).
|
||||
- The date comes from the resolved page date context.
|
||||
- Continuation lines belong to the preceding message.
|
||||
- The captured clock value is emitted with `Z`. Timezone metadata suppresses
|
||||
the missing-timezone warning but is not currently used for IANA conversion.
|
||||
|
||||
The required time and dash distinguish it from all existing bold-speaker
|
||||
formats:
|
||||
|
||||
- `**Speaker** (09:15): text` uses `bold-paren-time`.
|
||||
- `**Speaker** (9:15 AM): text` uses `bold-paren-time-12h`.
|
||||
- `**Speaker:** text` uses `bold-name-no-time`.
|
||||
- `**Speaker** (2026-04-09 9:15 AM): text` uses `imessage-slack`.
|
||||
|
||||
Keeping these examples in both `test_negative` and parser regression tests makes
|
||||
the non-shadowing contract executable.
|
||||
|
||||
## Adding a built-in format
|
||||
|
||||
1. Collect multiple anonymized examples, including separator and timestamp
|
||||
variants that occur in the same export family.
|
||||
2. Choose the narrowest grammar that represents the format. Constrain numeric
|
||||
fields such as hours and minutes when possible.
|
||||
3. Add at least two positive module-load samples and negative samples for every
|
||||
neighboring pattern that could plausibly overlap.
|
||||
4. Add parser tests that verify speakers, timestamps, text, continuation
|
||||
handling, and non-shadowing behavior.
|
||||
5. Add a dedicated JSONL fixture and include the same cases in
|
||||
`test/fixtures/conversation-formats/all.jsonl`.
|
||||
6. Run the focused parser tests and the fixture evaluator.
|
||||
7. Run the repository verification and full test suites before submission.
|
||||
8. Update `docs/architecture/KEY_FILES.md` when the registry count or supported
|
||||
format inventory changes.
|
||||
|
||||
Use generic fixture identities such as `Alice Example`, `Bob Example`, and
|
||||
`Summary Bot`. Never copy real transcript names or private content into source,
|
||||
tests, documentation, commits, or pull-request descriptions.
|
||||
@@ -75,6 +75,15 @@ Meta-pack stacking creator + investor + engineer via the v0.38
|
||||
preserved — this IS the active pack; the registry walks extends +
|
||||
borrow to materialize the merged view.
|
||||
|
||||
**Merge contract (T20 / #1749).** `resolvePack` merges parent → child
|
||||
(child-wins) for the six ingest/query-shaping fields: `page_types`,
|
||||
`link_types`, `frontmatter_links`, `enrichable_types`, `filing_rules`,
|
||||
and `takes_kinds` (unioned — a child cannot narrow it). `phases` and
|
||||
`calibration_domains` are **NOT** inherited: they gate cycle execution,
|
||||
so each pack must declare its own participation explicitly. That is why
|
||||
`gbrain-everything` re-declares all its phases and all 7
|
||||
`calibration_domains` — inheritance does not carry them.
|
||||
|
||||
Activate via `gbrain config set schema_pack gbrain-everything` and
|
||||
calibration_profile produces all 7 domain scorecards in one JSONB.
|
||||
|
||||
|
||||
@@ -56,7 +56,8 @@ that tuple lights up the `pack_upgrade_available` onboard check.
|
||||
│ gbrain onboard --check --explain shows per-cluster narrative │
|
||||
│ User reviews; if OK, runs: │
|
||||
│ gbrain jobs submit unify-types --allow-protected \ │
|
||||
│ --params '{"target_pack":"gbrain-base-v2"}' │
|
||||
│ --params '{"target_pack":"gbrain-base-v2","apply":true}' │
|
||||
│ (omit "apply":true for a dry-run; that is the default) │
|
||||
│ (Autopilot never auto-fires this; manual_only) │
|
||||
└──────────────────────────┬─────────────────────────────────────┘
|
||||
↓
|
||||
@@ -229,7 +230,7 @@ add `GBRAIN_AUDIT_FULL=1` (v0.43+ TODO; not yet wired).
|
||||
- Per-source pack-upgrade (the handler accepts `sourceId` but
|
||||
`findPackSuccessors` doesn't yet pass it through)
|
||||
- Cross-brain federated mounts that disagree on canonical packs
|
||||
- Automatic rollback (today: manual SQL or `gbrain pages restore`)
|
||||
- Automatic rollback (today: manual SQL or `gbrain restore`)
|
||||
- LLM-assisted mapping_rules codegen from production data (`gbrain
|
||||
schema detect-mappings`; deferred to v0.43+)
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ api_version: gbrain-schema-pack-v1
|
||||
name: my-pack
|
||||
version: 0.0.1
|
||||
gbrain_min_version: 0.39.0
|
||||
extends: gbrain-base # inherits everything from base; add overrides below
|
||||
extends: gbrain-base # inherits base's TYPES (see Merge contract below); add overrides
|
||||
description: |
|
||||
My personal pack.
|
||||
|
||||
@@ -170,6 +170,34 @@ enrichable_types: []
|
||||
filing_rules: []
|
||||
```
|
||||
|
||||
## Merge contract (`extends` + `borrow_from`)
|
||||
|
||||
`resolvePack` composes a pack against its `extends` chain (and any
|
||||
`borrow_from` targets) into the `resolved.manifest` every consumer reads
|
||||
(T20 / #1749). The rules:
|
||||
|
||||
- **Six fields inherit, child-wins:** `page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, and `takes_kinds`.
|
||||
A child value with the same key (type name, link name, etc.) overrides the
|
||||
parent's; keys the child doesn't declare come through from the parent.
|
||||
- **`page_types` ordering:** overrides of a base type keep the base's declared
|
||||
position (base's `inferType` prefix priority is authoritative); a genuinely
|
||||
new type — from the child, a `borrow_from`, or a middle pack in the chain —
|
||||
is prepended nearest-first, so a more-derived type's `path_prefix` wins
|
||||
regardless of how deep the chain is.
|
||||
- **`takes_kinds` is UNION, not replace** — it carries a Zod default, so an
|
||||
omitted field is indistinguishable from an explicit one. A child can ADD
|
||||
kinds but **cannot narrow** `takes_kinds` below base ∪ parent. If you need a
|
||||
smaller set, don't `extends` a pack that declares the larger one.
|
||||
- **`phases` and `calibration_domains` are NOT inherited** (child-only). They
|
||||
gate real cycle execution, so each pack must declare its own participation
|
||||
explicitly — inheriting them would silently make a child run phases it never
|
||||
requested. This is why `gbrain-everything` re-declares all its phases and
|
||||
calibration domains by hand. See `lens-packs.md` for the worked example.
|
||||
- **`borrow_from` is selective + non-transitive + fail-closed:** it pulls only
|
||||
the named `types`/`link_types` from the target's OWN declarations (omitting a
|
||||
category borrows none of it); a missing target throws `UnknownPackError`.
|
||||
|
||||
## Recovery + revert
|
||||
|
||||
The single-PR cathedral is hard to revert atomically. Per codex finding
|
||||
@@ -186,7 +214,7 @@ gbrain schema downgrade
|
||||
|
||||
1. `git revert <merge-commit>` — restores the code.
|
||||
2. `gbrain schema downgrade --to gbrain-base` — restores config.
|
||||
3. (Optional) `gbrain pages purge-deleted --older-than 0h` — drops
|
||||
3. (Optional) `gbrain purge-deleted --older-than 0h` — drops
|
||||
v0.39-typed pages that no longer have a matching type in the active
|
||||
pack.
|
||||
|
||||
|
||||
@@ -19,11 +19,13 @@ entire DB from scratch.
|
||||
|
||||
This means:
|
||||
|
||||
- **Disaster recovery is one command.** If your DB volume corrupts, if
|
||||
Postgres eats itself, if PGLite's WASM lock wedges — you don't need
|
||||
a backup. You wipe the DB, re-import from your brain repo, and the
|
||||
derived state regenerates. v0.32.3 ships `gbrain rebuild
|
||||
--confirm-destructive` as the documented one-liner.
|
||||
- **Disaster recovery is a short, boring sequence.** If your DB volume
|
||||
corrupts, if Postgres eats itself, if PGLite's WASM lock wedges — you
|
||||
don't need a backup. You wipe the derived tables (on PGLite,
|
||||
`gbrain reinit-pglite` wipes the whole embedded DB), re-import from
|
||||
your brain repo with `gbrain sync`, and `gbrain extract all`
|
||||
regenerates the derived state. See "Disaster recovery" below for the
|
||||
exact commands.
|
||||
- **Multi-machine sync is git.** Your brain is a repo. Push from one
|
||||
machine, pull from another, and the second machine's DB rebuilds on
|
||||
its next sync. No "back up the database" step.
|
||||
@@ -146,11 +148,9 @@ The promise the rule makes:
|
||||
# Snapshot what's there
|
||||
gbrain stats > /tmp/before.txt
|
||||
|
||||
# Wipe and rebuild
|
||||
gbrain rebuild --confirm-destructive # v0.32.3 — deletes derived tables
|
||||
# (pages + content_chunks survive
|
||||
# the CASCADE-safe design)
|
||||
# OR manually for v0.32.2:
|
||||
# Wipe and rebuild — delete the derived tables (pages + content_chunks
|
||||
# survive the CASCADE-safe design), then re-derive from the repo.
|
||||
# On PGLite, `gbrain reinit-pglite` wipes the whole embedded DB instead.
|
||||
psql -c 'DELETE FROM facts; DELETE FROM takes; DELETE FROM links; DELETE FROM timeline_entries;'
|
||||
gbrain sync
|
||||
gbrain extract all
|
||||
|
||||
@@ -76,7 +76,8 @@ gbrain onboard --check --explain # per-cluster narrative dry-run
|
||||
↓
|
||||
gbrain jobs submit unify-types \ # PROTECTED + manual_only
|
||||
--allow-protected \
|
||||
--params '{"target_pack":"gbrain-base-v2"}'
|
||||
--params '{"target_pack":"gbrain-base-v2","apply":true}'
|
||||
# omit "apply":true → dry-run (default)
|
||||
↓
|
||||
Handler runs 4 phases:
|
||||
┌─────────────────────────────────────┐
|
||||
@@ -108,8 +109,8 @@ Every primitive ships with a documented rollback:
|
||||
| Operation | Rollback |
|
||||
|-----------|----------|
|
||||
| Retype | `frontmatter.legacy_type = <original>` preserved on every page (D8). One SQL UPDATE restores types: `UPDATE pages SET type = frontmatter->>'legacy_type' WHERE frontmatter ? 'legacy_type'`. |
|
||||
| Page-to-link | Source page soft-deleted with 72h TTL. `gbrain pages restore <slug>` within 72h. Link row stays harmless if source restored. |
|
||||
| Page-to-alias | Source page soft-deleted with 72h TTL. `gbrain pages restore <slug>` within 72h. Alias row stays harmless (or `DELETE FROM slug_aliases WHERE alias_slug = <slug>` to clean up). |
|
||||
| Page-to-link | Source page soft-deleted with 72h TTL. `gbrain restore <slug>` within 72h. Link row stays harmless if source restored. |
|
||||
| Page-to-alias | Source page soft-deleted with 72h TTL. `gbrain restore <slug>` within 72h. Alias row stays harmless (or `DELETE FROM slug_aliases WHERE alias_slug = <slug>` to clean up). |
|
||||
| Active-pack flip | `gbrain schema use gbrain-base` reverses the flip. |
|
||||
|
||||
## What if my brain doesn't fit?
|
||||
@@ -127,7 +128,8 @@ For brains with substantial custom types that deserve their own canonical
|
||||
2. Edit your fork to add page_types + mapping_rules covering your
|
||||
custom domain.
|
||||
3. Target your fork: `gbrain jobs submit unify-types --allow-protected
|
||||
--params '{"target_pack":"my-pack"}'`
|
||||
--params '{"target_pack":"my-pack","apply":true}'` (omit `"apply":true`
|
||||
for a dry-run preview — that is the default)
|
||||
|
||||
Your fork can also declare `migration_from: {pack: gbrain-base-v2,
|
||||
version: "1.x"}` to register itself as a successor — future agents
|
||||
|
||||
@@ -154,10 +154,11 @@ these are the densest source of real bugs in the whole backlog.
|
||||
`aliases:`), first-H1-title, and basename fallback resolution (path-equality-only gives
|
||||
~5.5% edge recall on real vaults). Master shipped global-basename (#1388); the alias/
|
||||
title fallbacks are the still-novel part.
|
||||
- **Schema-pack-aware link extraction** (#1547, @billy-armstrong) — **OPEN, high.** The
|
||||
link extractor's `DIR_PATTERN` is a frozen 16-prefix const that ignores pack-declared
|
||||
`path_prefixes`, so default-pack installs silently lose wikilinks to `person/`,
|
||||
`writing/`, `wiki/*`. Resolve prefixes from the active pack.
|
||||
- **Schema-pack-aware link extraction** (#1547, @billy-armstrong) — **RESOLVED via #2576.**
|
||||
The extractor no longer gates on the frozen `DIR_PATTERN` whitelist: any dir-shaped
|
||||
path produces a candidate and the persist paths' page-existence checks decide, so
|
||||
pack-declared directories (`person/`, `writing/`, `wiki/*`, `ops/`) link without a
|
||||
prefix registry.
|
||||
- **DB-source extraction** (#1539, @afshaker) — **OPEN, high.** The cycle's extract phase
|
||||
only walks the filesystem, so DB-resident pages (imported transcripts, remote-DB brains)
|
||||
never get links/timeline and `brain_score` is capped. Thread `source:'db'`.
|
||||
|
||||
@@ -183,6 +183,6 @@ This also means the best AI agent setups will be open source by default. Closed,
|
||||
|
||||
Software distribution reimagined: the package is a markdown file, the runtime is a sufficiently smart model, the package manager is your AI agent, and the app store is a git repo.
|
||||
|
||||
`gbrain install voice-agent`
|
||||
`gbrain skillpack scaffold voice-agent`
|
||||
|
||||
That's it.
|
||||
|
||||
@@ -31,7 +31,7 @@ receipt file from disk and re-renders it. The other modes need the brain.
|
||||
| `--budget-usd N` | unset | Abort before next call's projected cost would exceed cap. Models without a `pricing.ts` entry fail loud (codex #4). |
|
||||
| `--source db|fs` | `db` | `fs` is reserved for v0.33+. |
|
||||
| `--slug-prefix P` | unset | Filter takes to pages whose slug starts with P. |
|
||||
| `--models a,b,c` | `openai:gpt-4o,anthropic:claude-opus-4-7,google:gemini-1.5-pro` | Comma-separated panel. |
|
||||
| `--models a,b,c` | `openai:gpt-5.2,anthropic:claude-opus-4-7,google:gemini-2.0-flash` | Comma-separated panel. |
|
||||
| `--json` | off | Emit the full receipt to stdout. |
|
||||
|
||||
## Receipt JSON shape (`schema_version: 1`)
|
||||
@@ -50,7 +50,7 @@ receipt file from disk and re-renders it. The other modes need the brain.
|
||||
},
|
||||
"prompt_sha8": "abcd1234",
|
||||
"models_sha8": "abcd1234",
|
||||
"models": ["openai:gpt-4o", "anthropic:claude-opus-4-7", "google:gemini-1.5-pro"],
|
||||
"models": ["openai:gpt-5.2", "anthropic:claude-opus-4-7", "google:gemini-2.0-flash"],
|
||||
"cycles_run": 3,
|
||||
"successes_per_cycle": [3, 3, 2],
|
||||
"verdict": "pass",
|
||||
|
||||
@@ -159,7 +159,8 @@ proxy for worker env.
|
||||
If a brain DB ever traverses a trust boundary, secrets stay out.
|
||||
- **Free-form names.** `inherit:` accepts any snake_case config-key on your
|
||||
worker — `database_url`, `anthropic_api_key`, `openai_api_key`,
|
||||
`voyage_api_key`, `groq_api_key`, `zeroentropy_api_key`, or any custom
|
||||
`openrouter_api_key`, `voyage_api_key`, `groq_api_key`,
|
||||
`zeroentropy_api_key`, or any custom
|
||||
field you stuff into `~/.gbrain/config.json`. The agent picks what it
|
||||
needs.
|
||||
- **`env:` still works** for non-secret values, or for cases where you
|
||||
|
||||
@@ -69,7 +69,7 @@ update_brain_page(slug, new_info, source):
|
||||
page = gbrain get {slug}
|
||||
|
||||
// TIMELINE: always APPEND (never edit existing entries)
|
||||
gbrain add_timeline_entry {slug} {
|
||||
gbrain timeline-add {slug} {
|
||||
date: today,
|
||||
summary: new_info.summary,
|
||||
detail: new_info.detail,
|
||||
|
||||
@@ -46,10 +46,10 @@ on user_shares_media(url_or_file):
|
||||
|
||||
# Step 4: Extract and cross-reference entities
|
||||
for person in transcript.mentioned_people:
|
||||
gbrain add_link <slug> <person_slug>
|
||||
gbrain add_link <person_slug> <slug>
|
||||
gbrain add_timeline_entry <person_slug> \
|
||||
--entry "Discussed in {video_title}: {what_was_said}" \
|
||||
gbrain link <slug> <person_slug>
|
||||
gbrain link <person_slug> <slug>
|
||||
gbrain timeline-add <person_slug> {date} \
|
||||
"Discussed in {video_title}: {what_was_said}" \
|
||||
--source "YouTube: {url}"
|
||||
|
||||
# PATTERN 2: Social Media Bundles
|
||||
@@ -80,8 +80,8 @@ on user_shares_media(url_or_file):
|
||||
|
||||
# Extract entities and cross-reference
|
||||
for entity in bundle.mentioned_entities:
|
||||
gbrain add_link <slug> <entity_slug>
|
||||
gbrain add_link <entity_slug> <slug>
|
||||
gbrain link <slug> <entity_slug>
|
||||
gbrain link <entity_slug> <slug>
|
||||
|
||||
# PATTERN 3: PDFs and Documents
|
||||
elif media.type == "pdf" or media.type == "document":
|
||||
@@ -109,8 +109,8 @@ on user_shares_media(url_or_file):
|
||||
"""
|
||||
|
||||
for entity in document.mentioned_entities:
|
||||
gbrain add_link <slug> <entity_slug>
|
||||
gbrain add_link <entity_slug> <slug>
|
||||
gbrain link <slug> <entity_slug>
|
||||
gbrain link <entity_slug> <slug>
|
||||
|
||||
# Always sync after ingestion
|
||||
gbrain sync
|
||||
@@ -127,7 +127,7 @@ on user_shares_media(url_or_file):
|
||||
## How to Verify
|
||||
|
||||
1. Ingest a YouTube video. Run `gbrain get media/youtube/{slug}`. Confirm the page has: the agent's analysis (not just a summary), key quotes with speaker attribution, and the full diarized transcript.
|
||||
2. Run `gbrain get_links media/youtube/{slug}`. Confirm back-links exist to brain pages for every person and company mentioned in the video.
|
||||
2. Run `gbrain call get_links '{"slug": "media/youtube/{slug}"}'`. Confirm back-links exist to brain pages for every person and company mentioned in the video.
|
||||
3. Pick a person mentioned in the video. Run `gbrain get <person_slug>`. Confirm their timeline has a new entry referencing the video with specific context.
|
||||
4. Ingest a tweet. Confirm the brain page includes the thread context, linked article summaries, and entity cross-references -- not just the tweet text.
|
||||
5. Run `gbrain search "{topic_from_video}"`. Confirm the media page appears in search results (verifies the content is indexed and searchable).
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# Embedding migration — moving a brain to another embedding provider
|
||||
|
||||
`gbrain migrate embeddings` re-embeds an entire brain onto a different
|
||||
embedding provider/model, safely and resumably. It is the forward path off a
|
||||
sunsetting provider (for example ZeroEntropy's hosted API, which shuts down
|
||||
2026-09-04 and is the shipped default for brains that never picked a model) —
|
||||
but it is provider-agnostic: any configured `provider:model` works as a
|
||||
target.
|
||||
|
||||
Also reachable as `gbrain retrieval-upgrade` (the name `doctor` and the
|
||||
README reference).
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# Preview the work + cost. Changes nothing.
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small --dry-run
|
||||
|
||||
# Run it (interactive confirm shows chunk count + $ estimate first).
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small
|
||||
|
||||
# Non-interactive (cron / scripts): --yes is required, else exit 2.
|
||||
gbrain migrate embeddings --to voyage:voyage-3-large --yes
|
||||
```
|
||||
|
||||
`--dim <N>` overrides the target width; it defaults to the provider recipe's
|
||||
declared width and is required for recipes that don't declare one (litellm,
|
||||
llama-server, and other bring-your-own-model providers).
|
||||
|
||||
## What it does, in order
|
||||
|
||||
1. **Plan.** Counts every chunk not already in the target embedding space —
|
||||
including chunks on pages with **no recorded embedding signature**
|
||||
(pages embedded before the v108 provenance stamp). Prices the re-embed
|
||||
from the pricing table; unknown providers print "estimate unavailable"
|
||||
instead of a fabricated number.
|
||||
2. **Consent gate.** Prints the plan; requires an interactive `y` or `--yes`.
|
||||
Non-TTY without `--yes` refuses with exit 2 (mirrors the `reindex-code`
|
||||
gate in [spend-controls](../operations/spend-controls.md)). Unlike the pure
|
||||
cost gates there, `spend.posture=tokenmax` does **not** bypass this one:
|
||||
posture waives the spend *ceiling*, and this gate also guards a
|
||||
destructive schema rebuild. Under `tokenmax` the dollar figure is marked
|
||||
informational and the confirmation is still asked. `--yes` is the single
|
||||
scripted bypass.
|
||||
3. **Live probe.** One tiny embed against the TARGET provider before any
|
||||
mutation — validates the API key, model id, and dimension support in a
|
||||
single call. A bad key fails here, with nothing changed.
|
||||
4. **Env-override gate.** Refuses when `GBRAIN_EMBEDDING_MODEL` /
|
||||
`GBRAIN_EMBEDDING_DIMENSIONS` would silently defeat the switch at
|
||||
runtime (the same guard `ze-switch` uses). `--ignore-env-override` for
|
||||
people running deliberate experiments.
|
||||
5. **Apply.** When the target width differs from the actual column width,
|
||||
runs the same atomic schema transition `ze-switch` uses, in one
|
||||
transaction. It rebuilds **all three dim-pinned text-embedding-space
|
||||
columns** — `content_chunks.embedding`, `query_cache.embedding`, and
|
||||
`facts.embedding` — at the new width, preserving each column's type
|
||||
(`vector` vs `halfvec`) and recreating its HNSW index. Missing any of the
|
||||
three leaves it silently broken: a narrow `query_cache.embedding` makes
|
||||
every cache write and read fail *by design* (the cache swallows errors so
|
||||
it can never break search) for a permanent 0% hit rate, and a narrow
|
||||
`facts.embedding` fails every per-fact embed write. The image/multimodal
|
||||
columns ARE deliberately untouched — they use separate models whose
|
||||
dimensions are independent of the text embedding model.
|
||||
Writes `embedding_model` + `embedding_dimensions` to BOTH config planes
|
||||
(file plane for the runtime gateway, DB plane for doctor), invalidates
|
||||
every chunk still in the old space — **including NULL-signature pages** —
|
||||
and purges the semantic query cache so stale cached results can't be
|
||||
served across the swap.
|
||||
6. **Re-embed.** The standard embed pipeline (`embed --stale --catch-up`)
|
||||
with per-source single-flight locks, rate-limit backoff, stderr progress,
|
||||
and optional DB-contention pacing (`--pace[=mode]`).
|
||||
|
||||
## What the rebuild deletes
|
||||
|
||||
The dimension change **deletes every stored embedding vector** in the brain —
|
||||
they are in the old model's space and unusable. They are not recoverable:
|
||||
going back to the previous provider means paying for a second full re-embed.
|
||||
`content_chunks` vectors are rebuilt by the re-embed pass, the query cache
|
||||
refills on the next query, and fact embeddings are rewritten on their next
|
||||
write (or a `gbrain extract` pass).
|
||||
|
||||
## Resume after a kill
|
||||
|
||||
The NULL-embedding column is the checkpoint. If the run is killed (or some
|
||||
pages fail to embed), re-run the **same command**: chunks already embedded on
|
||||
the target are never re-embedded, the schema/config steps no-op, and the run
|
||||
continues where it stopped. An in-flight marker (`embedding_migration.state`
|
||||
in DB config) records the target; it is cleared only when the backlog drains
|
||||
to zero.
|
||||
|
||||
A page whose chunks straddle two stale batches is embedded correctly but not
|
||||
stamped by the embed loop (which only stamps all-or-nothing per batch), so the
|
||||
migration runs one reconcile pass after the drain that stamps every
|
||||
fully-embedded page. Without it a large brain would report "incomplete" and the
|
||||
re-run would pay again for those pages. `--batch-size N` tunes the batch
|
||||
(default 2000).
|
||||
|
||||
`--no-embed` applies schema + config + invalidation and stops, so you can run
|
||||
the (potentially long) re-embed later or in the background:
|
||||
|
||||
```bash
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small --yes --no-embed
|
||||
gbrain embed --stale --catch-up --include-null-signature --background
|
||||
```
|
||||
|
||||
## During the migration
|
||||
|
||||
While the re-embed runs, semantic search returns degraded (lexical-arm-only)
|
||||
results for not-yet-re-embedded content. Pick a quiet window for large
|
||||
brains, or use `--pace` to keep the DB responsive.
|
||||
|
||||
## Pages without an embedding signature (#3391)
|
||||
|
||||
Pages embedded before provenance stamping have `embedding_signature IS NULL`
|
||||
and are grandfathered by the routine stale sweep (so an upgrade never
|
||||
surprise-re-embeds a whole corpus). After a provider swap that grandfather
|
||||
clause would silently leave those pages in the OLD embedding space — mixed
|
||||
vector spaces in one index, degrading retrieval with nothing in the logs.
|
||||
|
||||
- `gbrain migrate embeddings` always includes them.
|
||||
- Plain `gbrain embed --stale` warns when a model swap leaves NULL-signature
|
||||
pages behind, and `gbrain embed --stale --include-null-signature` re-embeds
|
||||
them.
|
||||
|
||||
## Reranker
|
||||
|
||||
Migrating embeddings does not touch the reranker. If
|
||||
`search.reranker.model` points at the outgoing provider, the plan prints a
|
||||
warning; disable it (`gbrain config set search.reranker.enabled false`) or
|
||||
point it at another provider.
|
||||
|
||||
## Self-hosting instead of migrating
|
||||
|
||||
If the outgoing model's weights are available (zembed-1's are Apache-2.0),
|
||||
serving them locally via `llama-server` / `ollama` / a LiteLLM proxy
|
||||
preserves your existing vectors — no re-embed at all. Point
|
||||
`embedding_model` at the local recipe and keep the same dimensions. The
|
||||
migration command is for when you'd rather move to a hosted provider.
|
||||
@@ -49,23 +49,23 @@ on enrich(entity, trigger):
|
||||
data["contacts"] = google_contacts(entity.email) # Contact data
|
||||
|
||||
# Step 5: Store raw data (auditable, re-processable)
|
||||
gbrain put_raw_data <entity_slug> \
|
||||
--data '{"sources": {"crustdata": {"fetched_at": "...", "data": {...}}, ...}}'
|
||||
gbrain call put_raw_data \
|
||||
'{"slug": "<entity_slug>", "data": {"sources": {"crustdata": {"fetched_at": "...", "data": {...}}, ...}}}'
|
||||
# Overwrite on re-enrichment, don't append
|
||||
|
||||
# Step 6: Write to brain page
|
||||
if path == "CREATE":
|
||||
gbrain put <entity_slug> --content "<compiled_truth_from_all_sources>"
|
||||
gbrain add_timeline_entry <entity_slug> --entry "Page created via enrichment"
|
||||
gbrain timeline-add <entity_slug> {date} "Page created via enrichment"
|
||||
elif path == "UPDATE":
|
||||
# Append timeline, update compiled truth ONLY if materially new
|
||||
gbrain add_timeline_entry <entity_slug> --entry "Enriched: {new_signal}"
|
||||
gbrain timeline-add <entity_slug> {date} "Enriched: {new_signal}"
|
||||
# Flag contradictions -- don't silently resolve them
|
||||
|
||||
# Step 7: Cross-reference the graph
|
||||
gbrain add_link <person_slug> <company_slug> # person -> company
|
||||
gbrain add_link <company_slug> <person_slug> # company -> person
|
||||
gbrain add_link <person_slug> <deal_slug> # person -> deal
|
||||
gbrain link <person_slug> <company_slug> # person -> company
|
||||
gbrain link <company_slug> <person_slug> # company -> person
|
||||
gbrain link <person_slug> <deal_slug> # person -> deal
|
||||
# Every entity page links to every other entity page that references it
|
||||
|
||||
# People page sections (not a LinkedIn profile -- a living portrait):
|
||||
@@ -94,8 +94,8 @@ on enrich(entity, trigger):
|
||||
## How to Verify
|
||||
|
||||
1. Enrich a Tier 1 person. Run `gbrain get <slug>` and confirm the page has Executive Summary, State, What They Believe, Contact, and Timeline sections populated from multiple sources.
|
||||
2. Run `gbrain get_raw_data <slug>`. Confirm raw API responses are stored with `sources.{provider}.fetched_at` timestamps.
|
||||
3. Run `gbrain get_links <slug>`. Confirm cross-reference links exist to the person's company page, deal pages, and related entities.
|
||||
2. Run `gbrain call get_raw_data '{"slug": "<slug>"}'`. Confirm raw API responses are stored with `sources.{provider}.fetched_at` timestamps.
|
||||
3. Run `gbrain call get_links '{"slug": "<slug>"}'`. Confirm cross-reference links exist to the person's company page, deal pages, and related entities.
|
||||
4. Check a page that was enriched AND has a user-written Assessment. Confirm the Assessment section was preserved, not overwritten by API data.
|
||||
5. Try to re-enrich the same person. Confirm the system checks the `fetched_at` timestamp and skips if less than a week old.
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ on upcoming_meeting(meeting):
|
||||
"last_interaction": page.timeline[0], # most recent
|
||||
"open_threads": page.open_threads,
|
||||
"relationship_temperature": page.relationship,
|
||||
"relevant_deals": gbrain get_links <attendee_slug>,
|
||||
"relevant_deals": gbrain call get_links '{"slug": "<attendee_slug>"}',
|
||||
}
|
||||
else:
|
||||
briefing[attendee] = "No brain page -- consider enriching"
|
||||
@@ -67,14 +67,14 @@ on inbox_cleared():
|
||||
for email in processed_emails:
|
||||
if email.contained_new_information:
|
||||
# Update the sender's brain page with new signal
|
||||
gbrain add_timeline_entry <sender_slug> \
|
||||
--entry "Email re: {subject}. Key info: {extracted_signal}" \
|
||||
gbrain timeline-add <sender_slug> {date} \
|
||||
"Email re: {subject}. Key info: {extracted_signal}" \
|
||||
--source "email from {sender} re {subject}, {date}"
|
||||
|
||||
# Update any mentioned entity pages too
|
||||
for entity in email.mentioned_entities:
|
||||
gbrain add_timeline_entry <entity_slug> \
|
||||
--entry "{what_was_said_about_them}" \
|
||||
gbrain timeline-add <entity_slug> {date} \
|
||||
"{what_was_said_about_them}" \
|
||||
--source "email from {sender}, {date}"
|
||||
|
||||
# WORKFLOW 4: Scheduling Nudges
|
||||
|
||||
@@ -21,14 +21,17 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
@@ -131,6 +134,16 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict.
|
||||
history rewrite still hard-blocks even with `--skip-failed`. Run
|
||||
`gbrain sync --skip-failed` to acknowledge a known-bad set yourself.
|
||||
|
||||
5. **Import checkpoints name the import target, not the caller's CWD.**
|
||||
Interrupted `gbrain import <dir>` runs may leave
|
||||
`~/.gbrain/import-checkpoint.json` so the next import can resume. The
|
||||
checkpoint `dir` is the absolute, resolved import target captured when
|
||||
import starts. It is not a cleanup instruction and it must not be
|
||||
re-derived from the process working directory. Checkpoints written by
|
||||
gbrain include `schema_version: 1`, `owner: "gbrain"`, and
|
||||
`kind: "import"` so downstream tools can validate the contract before
|
||||
deciding whether to resume.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Edit a file and search for the change.** Edit a brain markdown file,
|
||||
|
||||
@@ -32,15 +32,15 @@ on new_meeting_transcript(meeting):
|
||||
|
||||
# Step 3: Propagate to ALL entity pages (MANDATORY -- most agents skip this)
|
||||
for person in meeting.attendees + meeting.mentioned_people:
|
||||
gbrain add_timeline_entry <person_slug> \
|
||||
--entry "Met in '{meeting.title}' on {date}. Key points: ..." \
|
||||
gbrain timeline-add <person_slug> {date} \
|
||||
"Met in '{meeting.title}' on {date}. Key points: ..." \
|
||||
--source "Meeting notes '{meeting.title}', {date}"
|
||||
# Update their State section if new information surfaced
|
||||
# Update company pages for each person's company if relevant
|
||||
|
||||
for company in meeting.mentioned_companies:
|
||||
gbrain add_timeline_entry <company_slug> \
|
||||
--entry "Discussed in '{meeting.title}': {what_was_said}" \
|
||||
gbrain timeline-add <company_slug> {date} \
|
||||
"Discussed in '{meeting.title}': {what_was_said}" \
|
||||
--source "Meeting notes '{meeting.title}', {date}"
|
||||
|
||||
# Step 4: Extract action items
|
||||
@@ -49,8 +49,8 @@ on new_meeting_transcript(meeting):
|
||||
|
||||
# Step 5: Back-link everything (bidirectional graph)
|
||||
for entity in all_entities_mentioned:
|
||||
gbrain add_link <slug> <entity_slug> # meeting -> entity
|
||||
gbrain add_link <entity_slug> <slug> # entity -> meeting
|
||||
gbrain link <slug> <entity_slug> # meeting -> entity
|
||||
gbrain link <entity_slug> <slug> # entity -> meeting
|
||||
|
||||
# Step 6: Sync so new pages are immediately searchable
|
||||
gbrain sync
|
||||
@@ -73,7 +73,7 @@ on new_meeting_transcript(meeting):
|
||||
1. After ingesting a meeting, run `gbrain get meetings/{date}-{slug}`. Confirm the page has the agent's analysis above the bar and the full diarized transcript below it.
|
||||
2. For each attendee, run `gbrain get <attendee_slug>`. Check that their timeline has a new entry referencing the meeting with specific insights (not just "attended meeting").
|
||||
3. Pick a company mentioned in the meeting. Run `gbrain get <company_slug>`. Confirm a timeline entry exists referencing what was discussed about the company.
|
||||
4. Run `gbrain get_links meetings/{date}-{slug}`. Verify back-links exist to all attendee and entity pages.
|
||||
4. Run `gbrain call get_links '{"slug": "meetings/{date}-{slug}"}'`. Verify back-links exist to all attendee and entity pages.
|
||||
5. Run `gbrain search "{meeting_topic}"`. Confirm the meeting page appears in search results (verifies sync ran).
|
||||
|
||||
---
|
||||
|
||||
@@ -155,6 +155,7 @@ child-spawn time:
|
||||
- `inherit: ["database_url"]` → child env `GBRAIN_DATABASE_URL`
|
||||
- `inherit: ["anthropic_api_key"]` → child env `ANTHROPIC_API_KEY`
|
||||
- `inherit: ["openai_api_key"]` → child env `OPENAI_API_KEY`
|
||||
- `inherit: ["openrouter_api_key"]` → child env `OPENROUTER_API_KEY`
|
||||
- `inherit: ["voyage_api_key"]` → child env `VOYAGE_API_KEY`
|
||||
- `inherit: ["groq_api_key", "zeroentropy_api_key"]` → both injected
|
||||
- Or any arbitrary config-key your worker has (`my_custom_field` →
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Multi-language full-text search
|
||||
|
||||
GBrain's keyword search arm uses Postgres full-text search (tsvector/tsquery).
|
||||
The tokenizer language is configurable via the `GBRAIN_FTS_LANGUAGE`
|
||||
environment variable. Default: `english`.
|
||||
|
||||
## How it works
|
||||
|
||||
Postgres text-search configurations control stemming and stop-word removal.
|
||||
`GBRAIN_FTS_LANGUAGE` is read by `src/core/fts-language.ts` and applied on
|
||||
both sides of the search:
|
||||
|
||||
- **Query side** — `websearch_to_tsquery('<lang>', $query)` in both engines
|
||||
(Postgres and PGLite).
|
||||
- **Write side** — the `update_page_search_vector` and
|
||||
`update_chunk_search_vector` trigger functions that populate
|
||||
`pages.search_vector` and `content_chunks.search_vector`.
|
||||
|
||||
The value is validated against `/^[a-z][a-z0-9_]*$/` before it is ever
|
||||
interpolated into SQL (tsvector functions don't accept parameterized config
|
||||
names). Invalid values fall back to `english` with a warning.
|
||||
|
||||
## Built-in languages
|
||||
|
||||
Set the env var to any configuration your Postgres instance ships:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
export GBRAIN_FTS_LANGUAGE=spanish
|
||||
export GBRAIN_FTS_LANGUAGE=german
|
||||
```
|
||||
|
||||
List what's available:
|
||||
|
||||
```sql
|
||||
SELECT cfgname FROM pg_ts_config;
|
||||
```
|
||||
|
||||
PGLite (the embedded default engine) ships the same built-in snowball
|
||||
configurations as stock Postgres.
|
||||
|
||||
## First install vs. changing language later
|
||||
|
||||
On first install (or upgrade), the `configurable_fts_language` schema
|
||||
migration reads `GBRAIN_FTS_LANGUAGE` and stamps the trigger functions with
|
||||
that language. After the migration has run, changing the env var alone does
|
||||
NOT retokenize existing rows — the migration shows as applied and is skipped.
|
||||
Use the explicit command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview: language + row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command recreates both trigger functions under the new language and
|
||||
backfills every existing `pages` and `content_chunks` row in batches,
|
||||
streaming progress to stderr. It is idempotent: re-running with the same
|
||||
language produces identical vectors. `--json` prints a machine-readable
|
||||
result envelope but still requires `--yes` (or an interactive confirm).
|
||||
|
||||
No cache purge is needed. The resolved language is part of the query-cache
|
||||
key, so rows written under the previous language are unreachable after the
|
||||
switch — searches read the retokenized index immediately instead of being
|
||||
served pre-switch results for up to `search.cache.ttl_seconds`. Switching
|
||||
back reaches the original rows rather than rebuilding them.
|
||||
|
||||
## Recipe: accent-insensitive Portuguese (`pt_br`)
|
||||
|
||||
Brazilian Portuguese content often mixes accented and unaccented spellings
|
||||
("São Paulo" vs "Sao Paulo"). Build a custom config that folds accents via
|
||||
the `unaccent` extension, then stems with the portuguese snowball dictionary:
|
||||
|
||||
```sql
|
||||
CREATE EXTENSION IF NOT EXISTS unaccent;
|
||||
|
||||
CREATE TEXT SEARCH CONFIGURATION pt_br (COPY = portuguese);
|
||||
|
||||
ALTER TEXT SEARCH CONFIGURATION pt_br
|
||||
ALTER MAPPING FOR hword, hword_part, word
|
||||
WITH unaccent, portuguese_stem;
|
||||
```
|
||||
|
||||
Then point GBrain at it:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br
|
||||
gbrain reindex-search-vector --yes
|
||||
```
|
||||
|
||||
Note: custom configurations require a real Postgres instance (e.g. the
|
||||
Supabase engine). The config must exist BEFORE the migration or the reindex
|
||||
command runs, or Postgres will reject the trigger recreation with
|
||||
`text search configuration "pt_br" does not exist`.
|
||||
|
||||
## Caveats
|
||||
|
||||
- One language per brain: the setting is global to the database, not
|
||||
per-source. Mixed-language brains should pick the dominant language (the
|
||||
vector-search arm is language-agnostic and covers the rest).
|
||||
- Keep `GBRAIN_FTS_LANGUAGE` set consistently in every environment that
|
||||
writes to the brain (CLI shells, MCP server, cron jobs) — a writer without
|
||||
the env var tokenizes new rows in `english` until the next reindex.
|
||||
@@ -91,7 +91,7 @@ first):
|
||||
6. The seeded `default` source.
|
||||
|
||||
So inside `~/.gstack/plans/` on a brain that pinned `gstack` to
|
||||
`~/.gstack` via `.gbrain-source`, `gbrain put-page` implicitly writes to
|
||||
`~/.gstack` via `.gbrain-source`, `gbrain put` implicitly writes to
|
||||
the `gstack` source. Outside any registered directory with no env/dotfile
|
||||
set, it writes to the default.
|
||||
|
||||
@@ -114,8 +114,11 @@ Flip later with `gbrain sources federate <id>` / `unfederate <id>`.
|
||||
Full subcommand reference:
|
||||
|
||||
```
|
||||
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated]
|
||||
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated] [--force]
|
||||
Register a source. id: [a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?
|
||||
--path must be a git repo (or a subdirectory of one) — see
|
||||
"The git requirement for --path sources" below. --force
|
||||
skips that check to register before git-init exists.
|
||||
gbrain sources list [--json] List all sources with page counts + federation state.
|
||||
gbrain sources remove <id> [--yes] [--dry-run] [--keep-storage]
|
||||
Cascade-delete a source (pages, chunks, timeline).
|
||||
@@ -128,6 +131,47 @@ gbrain sources federate <id>
|
||||
gbrain sources unfederate <id>
|
||||
```
|
||||
|
||||
## The git requirement for --path sources
|
||||
|
||||
Every `--path` source must be a git repository (or live inside one — a
|
||||
subdirectory of a git repo works too) with at least one committed, tracked
|
||||
file under that path. `gbrain sources add` validates this at registration
|
||||
time and refuses a directory that doesn't qualify — no `.git` at all, a
|
||||
`git init` with no commit yet, or a commit made before `git add` — with an
|
||||
actionable error instead of silently registering a source that will fail
|
||||
(or worse, "succeed" while importing nothing) on its first `gbrain sync`.
|
||||
Fix it with:
|
||||
|
||||
```bash
|
||||
git -C <path> init
|
||||
git -C <path> add -A
|
||||
git -C <path> commit -m "initial import"
|
||||
gbrain sources add <id> --path <path>
|
||||
```
|
||||
|
||||
Two details that are easy to miss:
|
||||
|
||||
- **Files must actually be committed, not just present.** The sync walker
|
||||
reads files through git objects, so `git init` alone — even followed by an
|
||||
empty commit (`git commit --allow-empty`) — isn't enough. Registration
|
||||
checks for real tracked content (`git ls-tree HEAD` scoped to the path),
|
||||
not just a resolvable `HEAD`, so this footgun is caught immediately
|
||||
instead of surfacing later as a sync that imports nothing.
|
||||
- **`--force` registers the source anyway**, skipping the check. Use this if
|
||||
you're registering a path before an automated pipeline gets around to
|
||||
`git init`-ing it. GBrain never auto-`git init`s a `--path` source for
|
||||
you — it's your directory, not a gbrain-managed clone (same consent
|
||||
boundary as sync-time self-heal, which also never mutates a `--path`
|
||||
source without an explicit ask).
|
||||
|
||||
**If sync ever reports a problem with the sync anchor** (`last_commit`) —
|
||||
after a force-push, a history rewrite, or a from-scratch `git init` on a
|
||||
directory that was synced before — you do not need to reset anything by
|
||||
hand. `gbrain sync` detects an unreachable or non-ancestor anchor
|
||||
automatically and recovers: either a full reimport (anchor object missing)
|
||||
or a direct tree-to-tree diff against the orphaned bookmark (anchor present
|
||||
but rewritten), advancing the anchor to the new HEAD when it completes.
|
||||
|
||||
## Citation format for agents
|
||||
|
||||
When agents receive multi-source results they MUST cite pages in
|
||||
@@ -144,10 +188,10 @@ citations keep working.
|
||||
|
||||
```bash
|
||||
# Pass --source explicitly
|
||||
gbrain put-page topics/ai ... --source wiki
|
||||
gbrain put topics/ai ... --source wiki
|
||||
|
||||
# Or rely on the dotfile / env / CWD match
|
||||
cd ~/.gstack && gbrain put-page plans/multi-repo ...
|
||||
cd ~/.gstack && gbrain put plans/multi-repo ...
|
||||
# → source auto-resolves to gstack
|
||||
```
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ on every_inbound_message(message):
|
||||
for entity in entities:
|
||||
existing = gbrain search "{entity.name}"
|
||||
if existing:
|
||||
gbrain add_timeline_entry <entity_slug> \
|
||||
--entry "{what_was_said}" \
|
||||
gbrain timeline-add <entity_slug> {date} \
|
||||
"{what_was_said}" \
|
||||
--source "User, direct message, {timestamp}"
|
||||
# else: flag for enrichment if important enough
|
||||
|
||||
@@ -64,13 +64,13 @@ on nightly_schedule("02:00"):
|
||||
# The brain COMPOUNDS overnight.
|
||||
|
||||
# 5a: Entity sweep -- find unlinked mentions
|
||||
pages = gbrain list_pages
|
||||
pages = gbrain list
|
||||
for page in pages:
|
||||
mentions = extract_entity_mentions(page.content)
|
||||
existing_links = gbrain get_links <page.slug>
|
||||
existing_links = gbrain call get_links '{"slug": "<page.slug>"}'
|
||||
for mention in mentions:
|
||||
if mention not in existing_links:
|
||||
gbrain add_link <page.slug> <mention_slug> # fix broken graph
|
||||
gbrain link <page.slug> <mention_slug> # fix broken graph
|
||||
|
||||
# 5b: Citation audit -- find facts without sources
|
||||
for page in pages:
|
||||
@@ -80,7 +80,7 @@ on nightly_schedule("02:00"):
|
||||
|
||||
# 5c: Memory consolidation -- update compiled truth from timeline
|
||||
for page in stale_pages(older_than="7d"):
|
||||
timeline = gbrain get_timeline <page.slug>
|
||||
timeline = gbrain timeline <page.slug>
|
||||
if timeline.has_new_entries_since_last_consolidation:
|
||||
# Re-synthesize compiled truth from accumulated timeline
|
||||
updated_truth = consolidate(page.compiled_truth, timeline.new_entries)
|
||||
@@ -110,11 +110,11 @@ on nightly_schedule("02:00"):
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Send a message mentioning a person with a brain page. Confirm the agent detects the entity and adds a timeline entry to their page (`gbrain get_timeline <slug>`).
|
||||
1. Send a message mentioning a person with a brain page. Confirm the agent detects the entity and adds a timeline entry to their page (`gbrain timeline <slug>`).
|
||||
2. Ask the agent about someone in the brain. Confirm it runs `gbrain search` or `gbrain get` BEFORE reaching for external APIs (check the tool call order).
|
||||
3. Write a new page with `gbrain put`, then immediately run `gbrain search` for it. Confirm it appears in results (verifies sync ran).
|
||||
4. Run `gbrain doctor`. Confirm it returns a health report with database status, page count, and any flagged issues.
|
||||
5. After a dream cycle runs, check a page that had unlinked entity mentions. Confirm new links were added (`gbrain get_links <slug>`).
|
||||
5. After a dream cycle runs, check a page that had unlinked entity mentions. Confirm new links were added (`gbrain call get_links '{"slug": "<slug>"}'`).
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
|
||||
@@ -47,8 +47,8 @@ on user_message(message):
|
||||
|
||||
# Step 3: Cross-link to everything that shaped the thinking
|
||||
for entity in idea.influences:
|
||||
gbrain add_link originals/{slug} <entity_slug>
|
||||
gbrain add_link <entity_slug> originals/{slug}
|
||||
gbrain link originals/{slug} <entity_slug>
|
||||
gbrain link <entity_slug> originals/{slug}
|
||||
|
||||
# Step 4: Sync
|
||||
gbrain sync
|
||||
@@ -79,7 +79,7 @@ on user_message(message):
|
||||
|
||||
1. Generate an original idea in conversation (e.g., "I call this the 'ambition debt' problem -- every year you delay going big, the compound interest works against you"). Confirm a new page appears at `brain/originals/ambition-debt` with `gbrain get originals/ambition-debt`.
|
||||
2. Check that the page uses the user's exact phrasing for the title and slug -- not a sanitized version.
|
||||
3. Run `gbrain get_links originals/ambition-debt`. Confirm cross-links exist to related people, meetings, or other originals.
|
||||
3. Run `gbrain call get_links '{"slug": "originals/ambition-debt"}'`. Confirm cross-links exist to related people, meetings, or other originals.
|
||||
4. Express a take on someone else's idea (e.g., "I think Thiel's contrarian question is wrong because..."). Confirm it goes to `originals/` (synthesis is original), not `concepts/`.
|
||||
5. Run `gbrain search "ambition debt"`. Confirm the originals page appears in search results and is discoverable.
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ expect it.
|
||||
| `version` | string | yes | Your plugin's semver. Informational. |
|
||||
| `plugin_version` | string | yes | Contract lock. Must equal `"gbrain-plugin-v1"` for v0.15. |
|
||||
| `subagents` | string | no | Subdir name (default `subagents`). Escape-attempts are rejected. |
|
||||
| `description` | string | no | Shown in future `gbrain plugin list`. |
|
||||
| `description` | string | no | Shown in a future plugin-listing command. |
|
||||
|
||||
## Subagent definition files
|
||||
|
||||
|
||||
@@ -131,7 +131,9 @@ into gbrain so other clients can scaffold it. Default behavior:
|
||||
`~/.gbrain/harvest-private-patterns.txt` plus built-in defaults
|
||||
(canonical private fork name, common email regex, Slack channel pattern). Any
|
||||
match → rollback (delete the harvested files) and exit non-zero.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted. Harvest must preserve
|
||||
the top-level OpenClaw-native plugin fields (`id`, `configSchema`, `contracts`)
|
||||
because OpenClaw validates those before it can install the package.
|
||||
- `--no-lint` bypasses the linter (after a manual editorial scrub).
|
||||
|
||||
Use the `skillpack-harvest` skill (its companion editorial workflow)
|
||||
|
||||
@@ -44,6 +44,7 @@ These require manual setup (no self-installing recipe yet):
|
||||
|-------|-------------|
|
||||
| [Credential Gateway](credential-gateway.md) | Set up ClawVisor or Hermes for Gmail, Calendar, Contacts access |
|
||||
| [Meeting & Call Webhooks](meeting-webhooks.md) | Circleback meeting transcripts + Quo/OpenPhone SMS/calls |
|
||||
| [qm Harness](qm-harness.md) | gbrain as the company brain for a qm (multi-user agent harness) deployment — central HTTP MCP, per-scope clients, roster provisioning, write fencing |
|
||||
|
||||
## How to Read a Recipe
|
||||
|
||||
@@ -69,6 +70,12 @@ health_checks: # typed DSL to verify the integration is working
|
||||
auth_user: "$TWILIO_ACCOUNT_SID"
|
||||
auth_token: "$TWILIO_AUTH_TOKEN"
|
||||
label: "Twilio account"
|
||||
- type: heartbeat_max_age # staleness gate: FAILS `integrations doctor`
|
||||
max_age: 48h # when the newest heartbeat event is older.
|
||||
label: "Data freshness" # The other types are point-in-time and stay
|
||||
# green even when a sense stops producing data.
|
||||
output_paths: # repo-relative dirs the collector writes files to;
|
||||
- daily/voice/ # lets doctor/sync warn if one lands in db_only
|
||||
setup_time: 30 min # estimated time to complete setup
|
||||
---
|
||||
|
||||
@@ -86,7 +93,8 @@ a source install, or the global install copy) are trusted. Recipes discovered at
|
||||
runtime from `$GBRAIN_RECIPES_DIR` or a cwd-local `./recipes/` are marked untrusted:
|
||||
they cannot run `command` health checks, cannot run `http` health checks (SSRF
|
||||
defense), and cannot use the deprecated string health_check form. Untrusted recipes
|
||||
can still use `env_exists` and `any_of` compositions. To ship a recipe that runs
|
||||
can still use `env_exists`, `heartbeat_max_age` (reads only the local heartbeat
|
||||
file — no exec, no network), and `any_of` compositions. To ship a recipe that runs
|
||||
live checks, contribute it upstream so it becomes package-bundled.
|
||||
|
||||
## The Deterministic Collector Pattern
|
||||
|
||||
@@ -103,7 +103,7 @@ For GCP service-account / Vertex AI auth (production deployments), see the v0.32
|
||||
|
||||
### OpenRouter
|
||||
|
||||
Single OpenAI-compatible API for fan-out to OpenAI, Anthropic, Google, DeepSeek, Meta Llama, Qwen, and dozens of other hosted providers. One key, many models. Set `OPENROUTER_API_KEY` and use `openrouter:<provider>/<model>` (e.g. `openrouter:openai/gpt-5.2`, `openrouter:anthropic/claude-sonnet-4.6`).
|
||||
Single OpenAI-compatible API for fan-out to OpenAI, Anthropic, Google, DeepSeek, Meta Llama, Qwen, and dozens of other hosted providers. One key, many models. Set `OPENROUTER_API_KEY` or `openrouter_api_key` in `~/.gbrain/config.json`, then use `openrouter:<provider>/<model>` (e.g. `openrouter:openai/gpt-5.2`, `openrouter:anthropic/claude-sonnet-4.6`).
|
||||
|
||||
**Embedding**: `openai/text-embedding-3-small` (1536d default, Matryoshka shrink to 512/768/1024). OR's embedding catalog also includes `text-embedding-3-large`, `google/gemini-embedding-2-preview`, `qwen/qwen3-embedding-8b`, `bge-m3` — opt in via `--embedding-model openrouter:<id>`. Pricing matches the upstream provider (OR adds a small markup).
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
name: gbrain
|
||||
description: Search and write the company knowledge brain. Use for any question about the org, people, projects, decisions, or history, and to persist durable knowledge beyond this scope's notebook.
|
||||
---
|
||||
|
||||
# gbrain — the company brain
|
||||
|
||||
This sandbox has the `gbrain` CLI connected (thin-client) to the org's central
|
||||
brain. It is the deep, indexed, cross-source memory: org docs, shared channel
|
||||
knowledge, and every agent's durable notes. Your scope's own notebook stays the
|
||||
fast per-turn memory; the brain is where knowledge outlives a scope and becomes
|
||||
searchable by everyone entitled to it.
|
||||
|
||||
## First-run setup (once per sandbox — skip if `gbrain remote doctor` passes)
|
||||
|
||||
Your scope's brain credentials arrive via the deployment's secret handoff
|
||||
(keychain entry or one-time secret drop named `gbrain`). Then:
|
||||
|
||||
```bash
|
||||
gbrain init --mcp-only \
|
||||
--issuer-url "https://brain.<org>.com" \
|
||||
--mcp-url "https://brain.<org>.com/mcp" \
|
||||
--oauth-client-id "<client id from the handoff>" \
|
||||
--oauth-client-secret "<client secret from the handoff>"
|
||||
gbrain whoami # must succeed before using any other command
|
||||
```
|
||||
|
||||
Pass the secret with `--oauth-client-secret`, not via `GBRAIN_REMOTE_CLIENT_SECRET`:
|
||||
an env-sourced secret is deliberately NOT written to `~/.gbrain/config.json`, so
|
||||
every later command would fail with "No client_secret available" once the
|
||||
variable is out of scope. The flag persists it to the config file on this
|
||||
sandbox's durable disk, which is what the tool's credential capture expects.
|
||||
|
||||
Do not run `gbrain remote doctor` — it needs `admin` scope, which your client
|
||||
does not have (by design). `gbrain whoami` is the read-scope health check.
|
||||
|
||||
## Reading (do this liberally)
|
||||
|
||||
```bash
|
||||
gbrain search "who decided X and why" # hybrid semantic + keyword search
|
||||
gbrain get <slug> # read one page
|
||||
gbrain query "question" --json # search tuned for agent consumption
|
||||
```
|
||||
|
||||
You can read: the shared agent-memory source, org read-only sources (wiki,
|
||||
handbook), and everything under them. Reads are isolation-enforced server-side;
|
||||
you only ever see sources your client is entitled to.
|
||||
|
||||
## Writing (durable knowledge only, under YOUR prefixes)
|
||||
|
||||
Your client is write-fenced to slug prefixes — your own namespace plus the
|
||||
channels you belong to. Writes outside them are rejected server-side.
|
||||
|
||||
```bash
|
||||
# personal durable memory (your namespace):
|
||||
gbrain put emp-<your-slug>/people/jane-example --content "..."
|
||||
|
||||
# shared channel knowledge (channels you are in):
|
||||
gbrain put chan-eng/decisions/2026-08-database-choice --content "..."
|
||||
```
|
||||
|
||||
Conventions:
|
||||
- Write conclusions and durable facts, not chat transcripts. One page per
|
||||
entity/decision/topic; update the page rather than appending near-duplicates.
|
||||
- Markdown with YAML frontmatter; the brain chunks, embeds, and links it.
|
||||
- Cross-reference liberally: `gbrain link <from> <to>` (from must be in your
|
||||
namespace; linking TO any readable page is fine).
|
||||
- When you learn something channel-relevant in personal work, mirror the
|
||||
conclusion into the channel prefix with a `(said in <where>)` provenance
|
||||
note.
|
||||
|
||||
## When to reach for the brain
|
||||
|
||||
- Any question about the org, a person, a project, a decision, or history →
|
||||
`gbrain search` FIRST, then answer.
|
||||
- You produced knowledge with value beyond this conversation → `gbrain put`.
|
||||
- Something looks wrong (auth errors, empty results you don't expect) →
|
||||
`gbrain whoami` to confirm which client and scopes you're using, and report
|
||||
its output.
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env bash
|
||||
# provision-scopes.sh — roster-driven gbrain provisioning for a qm deployment
|
||||
# (or any multi-user agent harness with per-person + per-channel scopes).
|
||||
#
|
||||
# Reads a roster of channels + employees and converges the brain to it:
|
||||
# - ensures the shared agent-memory source exists (path-less: agents write
|
||||
# pages into it over MCP; `gbrain sync` skips it; if the brain host has
|
||||
# sync.repo_path configured, pages also write through to .sources/<id>/
|
||||
# on disk for git-backed durability)
|
||||
# - registers one OAuth client per employee, write-fenced via
|
||||
# bound_slug_prefixes to emp-<slug>/ plus chan-<c>/ for each channel
|
||||
# they are in, with federated reads over the memory source + any
|
||||
# read-only sources you pass
|
||||
# - re-running after roster edits rescopes existing clients IN PLACE
|
||||
# (client ids are remembered in the state file; secrets never rotate
|
||||
# unless you revoke + delete the state row)
|
||||
#
|
||||
# Usage:
|
||||
# provision-scopes.sh roster.tsv \
|
||||
# [--memory-source agents] [--read-sources org-wiki,handbook] \
|
||||
# [--budget-usd-per-day 5] [--state-file roster.state.tsv] \
|
||||
# [--secrets-out new-credentials.tsv] [--gbrain gbrain] [--dry-run]
|
||||
#
|
||||
# Roster format (one entry per line; '#' comments and blank lines ignored):
|
||||
# channel <slug>
|
||||
# employee <slug> [comma-separated channel slugs]
|
||||
#
|
||||
# SECURITY: --secrets-out receives client secrets for NEW registrations,
|
||||
# written exactly once (gbrain never re-shows them). Deliver each row to its
|
||||
# scope's sandbox (e.g. via the harness keychain or a one-time secret drop),
|
||||
# then delete the file.
|
||||
#
|
||||
# ponytail: sequential CLI loop, one gbrain invocation per roster row — fine
|
||||
# to hundreds of employees; batch via the admin API if that ever hurts.
|
||||
|
||||
# -f (noglob) is load-bearing, not stylistic: roster lines are word-split
|
||||
# unquoted below, so without it a line like `employee * eng` would expand
|
||||
# against the working directory and silently provision a filename as a
|
||||
# person — i.e. the wrong write fence. Nothing here needs globbing.
|
||||
set -euf -o pipefail
|
||||
|
||||
# Client secrets and the id state file are written by this script; 077 makes
|
||||
# them 0600 instead of the default 0644. Set before the first file is created.
|
||||
umask 077
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
# Slugs become source ids, client names, AND slug-prefix write fences. The
|
||||
# fence list is comma-separated, so an unvalidated slug containing a comma
|
||||
# would inject an EXTRA prefix and hand the client write access to someone
|
||||
# else's namespace. Fail closed on anything that isn't plain kebab-case.
|
||||
valid_slug() {
|
||||
case "$1" in
|
||||
'') return 1 ;;
|
||||
-*|*-) return 1 ;;
|
||||
*[!a-z0-9-]*) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
require_slug() {
|
||||
valid_slug "$2" || die "roster: invalid $1 slug '$2' (allowed: lowercase a-z, 0-9, interior hyphens)"
|
||||
}
|
||||
|
||||
ROSTER="${1:-}"
|
||||
[ -n "$ROSTER" ] && [ -f "$ROSTER" ] || die "usage: provision-scopes.sh <roster-file> [flags] (roster not found: '$ROSTER')"
|
||||
shift
|
||||
|
||||
GBRAIN="${GBRAIN:-gbrain}"
|
||||
MEMORY_SOURCE="agents"
|
||||
READ_SOURCES=""
|
||||
BUDGET="5"
|
||||
STATE_FILE=""
|
||||
SECRETS_OUT=""
|
||||
DRY_RUN=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--memory-source) MEMORY_SOURCE="$2"; shift 2 ;;
|
||||
--read-sources) READ_SOURCES="$2"; shift 2 ;;
|
||||
--budget-usd-per-day) BUDGET="$2"; shift 2 ;;
|
||||
--state-file) STATE_FILE="$2"; shift 2 ;;
|
||||
--secrets-out) SECRETS_OUT="$2"; shift 2 ;;
|
||||
--gbrain) GBRAIN="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
*) die "unknown flag: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
STATE_FILE="${STATE_FILE:-${ROSTER}.state.tsv}"
|
||||
SECRETS_OUT="${SECRETS_OUT:-${ROSTER}.new-credentials.tsv}"
|
||||
|
||||
# The roster usually lives in the deployment repo, so the default secrets and
|
||||
# state paths land there too — one `git add -A` from committing live
|
||||
# credentials. The STATE file matters as much as the secrets file: it maps
|
||||
# employee -> client_id, and this script feeds that id straight to
|
||||
# `rescope-client`, so whoever can write it decides which client receives a
|
||||
# given employee's write authority. Treat both as privileged infrastructure,
|
||||
# at the same trust level as the roster itself.
|
||||
for f in "$SECRETS_OUT" "$STATE_FILE"; do
|
||||
if git -C "$(dirname "$f")" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
echo "WARN: $f is inside a git work tree. Never commit it;" >&2
|
||||
echo " gitignore it, or pass --secrets-out/--state-file outside the repo." >&2
|
||||
fi
|
||||
done
|
||||
|
||||
# A group/world-writable parent directory defeats the symlink and ownership
|
||||
# checks below: anyone with write access there can swap the file between our
|
||||
# check and our append. Refuse rather than pretend the checks hold.
|
||||
for d in "$(dirname "$SECRETS_OUT")" "$(dirname "$STATE_FILE")"; do
|
||||
perms=$(ls -ld "$d" | awk '{print $1}')
|
||||
case "$perms" in
|
||||
?????w*|????????w*) die "refusing to write credentials into a group/world-writable directory: $d ($perms)" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Secure the credential sinks BEFORE anything is appended. umask only governs
|
||||
# files this script creates; a pre-existing world-readable file would receive
|
||||
# secrets first and be chmod'ed only afterwards, and a symlink planted at
|
||||
# either path would redirect them entirely.
|
||||
for f in "$SECRETS_OUT" "$STATE_FILE"; do
|
||||
[ -L "$f" ] && die "refusing to write credentials through a symlink: $f"
|
||||
if [ -e "$f" ]; then
|
||||
[ -f "$f" ] || die "refusing to write credentials to a non-regular file: $f"
|
||||
[ -O "$f" ] || die "refusing to write credentials to a file owned by another user: $f"
|
||||
else
|
||||
: > "$f"
|
||||
fi
|
||||
chmod 600 "$f"
|
||||
done
|
||||
|
||||
run() {
|
||||
if [ "$DRY_RUN" = 1 ]; then echo "DRY-RUN: $GBRAIN $*" >&2; return 0; fi
|
||||
# shellcheck disable=SC2086 — $GBRAIN may carry args ("bun run src/cli.ts")
|
||||
$GBRAIN "$@"
|
||||
}
|
||||
|
||||
state_lookup() { # state_lookup <employee-slug> -> client_id or empty
|
||||
[ -f "$STATE_FILE" ] || return 0
|
||||
awk -F'\t' -v s="$1" '$1 == s { print $2; exit }' "$STATE_FILE"
|
||||
}
|
||||
|
||||
# ── Pass 1: parse roster, collect declared channels ─────────────────────────
|
||||
CHANNELS=""
|
||||
EMPLOYEES=""
|
||||
lineno=0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
lineno=$((lineno + 1))
|
||||
line="${line%%#*}"
|
||||
line="${line%$'\r'}" # a CRLF roster would otherwise yield 'emp-alice\r/' prefixes that fence everything out
|
||||
[ -z "${line//[[:space:]]/}" ] && continue
|
||||
# shellcheck disable=SC2086 — deliberate word split; globbing is off (set -f above)
|
||||
set -- $line
|
||||
[ "$#" -le 3 ] || die "roster line $lineno: too many fields ('$line'). Channels are ONE comma-separated field with no spaces: 'employee alice eng,product'"
|
||||
case "$1" in
|
||||
channel)
|
||||
require_slug channel "${2:-}"
|
||||
CHANNELS="$CHANNELS $2"
|
||||
;;
|
||||
employee)
|
||||
require_slug employee "${2:-}"
|
||||
case " $EMPLOYEES " in *" $2:"*) die "roster line $lineno: employee '$2' listed twice" ;; esac
|
||||
if [ -n "${3:-}" ]; then
|
||||
for c in ${3//,/ }; do require_slug "channel-reference" "$c"; done
|
||||
fi
|
||||
EMPLOYEES="$EMPLOYEES $2:${3:-}"
|
||||
;;
|
||||
*) die "roster line $lineno: unknown entry type '$1' (expected 'channel' or 'employee')" ;;
|
||||
esac
|
||||
done < "$ROSTER"
|
||||
|
||||
# ── Pass 2: ensure the shared memory source exists (path-less) ──────────────
|
||||
if out=$(run sources add "$MEMORY_SOURCE" --name "agent memory ($MEMORY_SOURCE)" 2>&1); then
|
||||
echo "source '$MEMORY_SOURCE': created"
|
||||
else
|
||||
echo "$out" | grep -q "already registered" || die "sources add failed: $out"
|
||||
echo "source '$MEMORY_SOURCE': already exists"
|
||||
fi
|
||||
|
||||
# ── Pass 3: converge one client per employee ────────────────────────────────
|
||||
FED_READ="$MEMORY_SOURCE${READ_SOURCES:+,$READ_SOURCES}"
|
||||
new_secrets=0
|
||||
|
||||
for entry in $EMPLOYEES; do
|
||||
slug="${entry%%:*}"
|
||||
chans="${entry#*:}"
|
||||
|
||||
prefixes="emp-$slug/"
|
||||
if [ -n "$chans" ]; then
|
||||
for c in ${chans//,/ }; do
|
||||
echo " $CHANNELS " | grep -q " $c " || echo "WARN: employee '$slug' references undeclared channel '$c'" >&2
|
||||
prefixes="$prefixes,chan-$c/"
|
||||
done
|
||||
fi
|
||||
|
||||
client_id="$(state_lookup "$slug")"
|
||||
if [ -n "$client_id" ]; then
|
||||
# The state file usually sits in the deployment repo, so anyone who can
|
||||
# edit it could otherwise retarget this privileged rescope at an arbitrary
|
||||
# client id (e.g. point alice's row at an admin client). Shape-check it.
|
||||
case "$client_id" in
|
||||
gbrain_cl_) die "state file: empty client id for '$slug'" ;;
|
||||
gbrain_cl_*[!a-zA-Z0-9_]*) die "state file: malformed client id for '$slug': $client_id" ;;
|
||||
gbrain_cl_*) ;;
|
||||
*) die "state file: client id for '$slug' does not look like a gbrain client: $client_id" ;;
|
||||
esac
|
||||
# --source too, so a re-run actually CONVERGES the client to the roster:
|
||||
# without it, changing --memory-source (or inheriting a state row written
|
||||
# against an older one) silently leaves the old write source in place
|
||||
# while the script reports success.
|
||||
run auth rescope-client "$client_id" --source "$MEMORY_SOURCE" \
|
||||
--federated-read "$FED_READ" --bound-slug-prefixes "$prefixes" >/dev/null
|
||||
echo "employee '$slug': rescoped $client_id [write: $prefixes]"
|
||||
elif [ "$DRY_RUN" = 1 ]; then
|
||||
echo "employee '$slug': WOULD register qm-emp-$slug [write: $prefixes] [read: $FED_READ]"
|
||||
continue
|
||||
else
|
||||
out=$(run auth register-client "qm-emp-$slug" \
|
||||
--grant-types client_credentials --scopes "read write" \
|
||||
--source "$MEMORY_SOURCE" --federated-read "$FED_READ" \
|
||||
--bound-slug-prefixes "$prefixes" --budget-usd-per-day "$BUDGET" 2>&1) \
|
||||
|| die "register-client failed for '$slug' (output withheld: it can contain a secret). Re-run the command by hand to see why."
|
||||
client_id=$(echo "$out" | sed -n 's/.*Client ID:[[:space:]]*\(gbrain_cl_[^[:space:]]*\).*/\1/p' | head -1)
|
||||
secret=$(echo "$out" | sed -n 's/.*Client Secret:[[:space:]]*\(gbrain_cs_[^[:space:]]*\).*/\1/p' | head -1)
|
||||
if [ -z "$client_id" ] || [ -z "$secret" ]; then
|
||||
# The client may well have been created — dying silently would strand a
|
||||
# live credential nobody can find. Say so WITHOUT echoing the captured
|
||||
# output: it contains the freshly minted secret, and this path ends up
|
||||
# in CI logs.
|
||||
die "could not parse client id/secret for '$slug' from register-client output (output withheld: it contains a secret). A client MAY have been created; check \`gbrain auth list\` and revoke any stray 'qm-emp-$slug'."
|
||||
fi
|
||||
printf '%s\t%s\n' "$slug" "$client_id" >> "$STATE_FILE"
|
||||
printf '%s\t%s\t%s\n' "$slug" "$client_id" "$secret" >> "$SECRETS_OUT"
|
||||
chmod 600 "$STATE_FILE" "$SECRETS_OUT" 2>/dev/null || true # umask covers new files; this covers pre-existing ones
|
||||
new_secrets=$((new_secrets + 1))
|
||||
echo "employee '$slug': registered $client_id [write: $prefixes]"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Pass 4: flag offboarded employees ───────────────────────────────────────
|
||||
# Removing someone from the roster is the highest-stakes edit there is, and
|
||||
# this script cannot safely revoke on its own (a typo'd roster would nuke live
|
||||
# credentials). Report instead, with the exact command.
|
||||
if [ -f "$STATE_FILE" ]; then
|
||||
while IFS=$'\t' read -r st_slug st_client _rest; do
|
||||
[ -n "${st_slug:-}" ] || continue
|
||||
case " $EMPLOYEES " in
|
||||
*" $st_slug:"*) ;;
|
||||
*) echo "STALE: '$st_slug' ($st_client) is no longer in the roster but its credentials still work." >&2
|
||||
echo " Revoke with: $GBRAIN auth revoke-client $st_client" >&2 ;;
|
||||
esac
|
||||
done < "$STATE_FILE"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Done. State: $STATE_FILE"
|
||||
if [ "$new_secrets" -gt 0 ]; then
|
||||
echo "$new_secrets NEW client secret(s) written to $SECRETS_OUT — deliver to each scope's sandbox, then DELETE the file."
|
||||
fi
|
||||
@@ -0,0 +1,10 @@
|
||||
# Roster for provision-scopes.sh — one line per channel / employee.
|
||||
# channel <slug>
|
||||
# employee <slug> [comma-separated channels they belong to]
|
||||
|
||||
channel eng
|
||||
channel product
|
||||
|
||||
employee alice-example eng,product
|
||||
employee bob-example eng
|
||||
employee carol-example
|
||||
|
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"id": "gbrain",
|
||||
"label": "gbrain company brain",
|
||||
"advertise": "gbrain",
|
||||
"hints": [
|
||||
"Company knowledge brain: searchable, cross-source, persistent.",
|
||||
"Search it BEFORE answering questions about the org, people, projects, decisions, or history: `gbrain search \"<question>\"`.",
|
||||
"Write durable knowledge with `gbrain put <slug> --content ...`, only under your own slug prefixes.",
|
||||
"See the gbrain skill for slug conventions and first-run setup."
|
||||
],
|
||||
"auth": {
|
||||
"check": "gbrain whoami",
|
||||
"reauth": "gbrain init --mcp-only --force --issuer-url \"$GBRAIN_ISSUER_URL\" --mcp-url \"$GBRAIN_MCP_URL\" --oauth-client-id \"$GBRAIN_CLIENT_ID\" --oauth-client-secret \"$GBRAIN_CLIENT_SECRET\"",
|
||||
"credentialPaths": [
|
||||
{ "path": ".gbrain/config.json", "kind": "file" }
|
||||
]
|
||||
},
|
||||
"install": { "binary": "gbrain" }
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
# qm (multi-user agent harness) — gbrain as the company brain
|
||||
|
||||
Connect gbrain to [qm](https://github.com/yc-software/qm) — the multiplayer
|
||||
agent harness where each employee and each channel gets an isolated agent
|
||||
scope — so every scope's agent can search and write one shared, indexed,
|
||||
isolation-enforced company brain. The same recipe fits any harness with
|
||||
per-person sandboxes that can run a CLI.
|
||||
|
||||
**Shape:** one central `gbrain serve --http` (OAuth 2.1) next to qm's core;
|
||||
the `gbrain` binary baked into qm's sandbox image as a thin client; one OAuth
|
||||
client per employee, read-fenced by source federation and write-fenced by
|
||||
`bound_slug_prefixes`. Zero qm code changes — everything lives in the qm
|
||||
*deployment directory*.
|
||||
|
||||
qm's native memory (per-scope notebook) stays as-is for fast per-turn recall.
|
||||
gbrain adds what qm doesn't have: semantic + hybrid search, cross-scope
|
||||
knowledge, entity graphs, and durable memory that outlives a scope.
|
||||
|
||||
## Topology
|
||||
|
||||
| gbrain concept | qm concept |
|
||||
|---|---|
|
||||
| one brain (one Postgres/Supabase DB) | the org |
|
||||
| source `agents` (path-less, shared) | all agent-written memory |
|
||||
| slug prefix `emp-<slug>/` in `agents` | an employee's personal scope |
|
||||
| slug prefix `chan-<slug>/` in `agents` | a channel/room scope |
|
||||
| source `org-wiki` (git-backed, read-only) | company docs |
|
||||
| OAuth client `qm-emp-<slug>` | one employee's agent identity |
|
||||
|
||||
Isolation model:
|
||||
|
||||
- **Reads** are source-granular, SQL-enforced (`federated_read`): every
|
||||
employee client reads `agents` + the read-only sources you grant.
|
||||
- **Writes** are slug-prefix-granular, server-enforced (`bound_slug_prefixes`,
|
||||
v0.42.72.0+): a client can only mutate pages under its own `emp-<slug>/`
|
||||
and its channels' `chan-<x>/` prefixes — on `put_page`, `delete_page`,
|
||||
`restore_page`, `add_tag`, `remove_tag`, `add_link`/`remove_link`,
|
||||
`add_timeline_entry`, `revert_version` and `put_raw_data`, plus the
|
||||
`POST /ingest` webhook route. Not by convention.
|
||||
- **Every op that is not a plain read is denied unless allow-listed.** Ops
|
||||
that write by a key other than a slug — `extract_entities` and
|
||||
`extract_facts` (which mutate `people/*` and `companies/*`), `forget_fact`
|
||||
(targets a fact by numeric id, across sources), `ontology_propose`, and the
|
||||
`sources_admin` pair `sources_add`/`sources_remove` — cannot be fenced by
|
||||
slug, so a bound client gets `permission_denied` at dispatch. The gate keys
|
||||
on "not a pure read", not on a list of scope strings, so a write op added
|
||||
later (or one carrying a bespoke scope) is denied until it is explicitly
|
||||
fenced and added to `CLIENT_FENCED_WRITE_OPS` (`src/core/operations.ts`).
|
||||
`think` is allow-listed because remote callers cannot persist from it;
|
||||
`submit_agent` because it enforces this same column itself.
|
||||
- **Indirect write paths are gated too, not just the ops.** `put_page`'s
|
||||
facts backstop would otherwise extract entities from the page body and
|
||||
write fact rows (and a `## Facts` fence on git-backed sources) onto
|
||||
`people/*` pages the caller never named — the same capability
|
||||
`extract_facts` is denied for, reached through an in-prefix write. It is
|
||||
skipped for bound clients. `POST /ingest` is refused outright: its handler
|
||||
bypasses the op layer *and* discards the source grant for untrusted
|
||||
payloads, so it would write into the `default` source.
|
||||
### Known limitations — read these before you rely on the fence
|
||||
|
||||
The write fence is a **write** boundary within a source. It is not a privacy
|
||||
boundary, and it does not make every side effect prefix-clean. As of
|
||||
v0.42.73.2:
|
||||
|
||||
- **The fence follows a delegated write.** When a client with `agent` scope
|
||||
hands work to a subagent via `submit_agent`, that subagent runs under its own
|
||||
slug confinement rather than the parent's OAuth binding. Both confinements are
|
||||
enforced, including on the path where deduplication redirects a write onto an
|
||||
existing page: the redirected target is checked against whichever confinement
|
||||
the calling context actually carries, so delegation does not widen what a
|
||||
client can write.
|
||||
|
||||
- **`add_link`/`remove_link` fence the `from` endpoint only.** A bound client
|
||||
can create an edge pointing AT a page it cannot write; the edge's `context`
|
||||
text surfaces in that page's backlinks and contributes to its search
|
||||
ranking. Fencing `to` would break legitimate cross-referencing into
|
||||
`org-wiki`, so this is deliberate — treat inbound-edge context as untrusted
|
||||
content, the same way you treat page bodies.
|
||||
- **Reads are source-granular, never prefix-granular.** Everyone entitled to
|
||||
a source can read every prefix in it. If a scope needs genuine read
|
||||
privacy, give it its own source.
|
||||
- **`put_page` can create one reverse graph edge outside the fence.** If a
|
||||
page body cites a code location (`src/x.ts:42`) and a code page for it
|
||||
exists *in the same source*, doc↔impl reconciliation adds an edge
|
||||
originating from that code page. It affects graph/backlink ranking, not
|
||||
page content. Unreachable in the layout above (the `agents` source is
|
||||
path-less and holds no code pages); it applies only if you point employee
|
||||
writes at a code-synced source.
|
||||
- **A few read ops are still brain-wide** and ignore the federated grant:
|
||||
`get_recent_salience`, `find_anomalies`, `find_contradictions`, and
|
||||
`sources_list`/`sources_status` (which expose source ids, paths and URLs).
|
||||
A read-scoped client can learn facts derived from sources it was not
|
||||
granted. Pre-existing, not introduced by the fence; if that matters for
|
||||
your deployment, withhold those tools at the harness layer for now.
|
||||
- **Reads touch `last_retrieved_at`** on the pages they return, including
|
||||
pages in read-only sources. Freshness/usage signals are therefore
|
||||
writable-by-reading; nothing else about the page is.
|
||||
- **`POST /ingest` writes land in the `default` source** regardless of the
|
||||
calling client's `source_id`, because the handler discards the source for
|
||||
untrusted payloads. Bound clients are refused the route outright for this
|
||||
reason; if you point a webhook integration at it, scope that brain's
|
||||
`default` source deliberately.
|
||||
- **Tradeoff to state out loud:** read isolation is per-source, so within the
|
||||
shared `agents` source every employee can *read* every prefix (including
|
||||
other employees' `emp-*/`). That matches qm's transparent-by-default,
|
||||
everything-audited posture. If you need hard read privacy for personal
|
||||
memory, give those employees their own write source instead of a prefix
|
||||
(one `sources add emp-<slug>` + `--source emp-<slug>` per client) and keep
|
||||
channel prefixes in `agents` via a second, channels-only client — at the
|
||||
cost of two credentials in that sandbox.
|
||||
|
||||
## Host setup (the machine running qm's core, or any box its sandboxes can reach)
|
||||
|
||||
```bash
|
||||
# 1. Engine: Postgres/Supabase. PGLite is single-process and cannot serve
|
||||
# many concurrent sandboxes.
|
||||
gbrain init --supabase --embedding-model voyage:voyage-4-large
|
||||
|
||||
# 2. Modes + gates (publish_* default OFF and fail as silent 403s):
|
||||
gbrain config set search.mode balanced
|
||||
gbrain config set mcp.publish_skills true
|
||||
gbrain config set mcp.publish_advisor true
|
||||
|
||||
# 3. Read-only org sources + first sync:
|
||||
gbrain sources add org-wiki --path ~/brains/org-wiki
|
||||
gbrain sync --all # cron this
|
||||
|
||||
# 4. Serve over HTTP MCP (OAuth 2.1):
|
||||
gbrain serve --http --bind 0.0.0.0 --port 3131 \
|
||||
--public-url https://brain.acme-example.com
|
||||
```
|
||||
|
||||
Never hand sandboxes `DATABASE_URL` — direct DB access bypasses OAuth, source
|
||||
federation, and the write fence entirely.
|
||||
|
||||
## Provision scopes from a roster
|
||||
|
||||
[`qm-harness-snippets/provision-scopes.sh`](qm-harness-snippets/provision-scopes.sh)
|
||||
converges the brain to a roster file
|
||||
([`roster.example.tsv`](qm-harness-snippets/roster.example.tsv)):
|
||||
|
||||
```bash
|
||||
bash provision-scopes.sh roster.tsv --read-sources org-wiki
|
||||
```
|
||||
|
||||
- Creates the path-less `agents` source (agent-written memory needs no git
|
||||
clone; if the host has `sync.repo_path` configured, pages also write
|
||||
through to `.sources/agents/` for git-backed durability).
|
||||
- Registers `qm-emp-<slug>` clients: `--scopes "read write"`,
|
||||
`--source agents`, `--federated-read agents,org-wiki`,
|
||||
`--bound-slug-prefixes emp-<slug>/,chan-<a>/,...`, per-day budget.
|
||||
- **Idempotent:** re-run after every roster edit; existing clients are
|
||||
`rescope-client`ed in place (channel joins/leaves update the write fence
|
||||
without rotating secrets).
|
||||
- New client secrets land once in `<roster>.new-credentials.tsv` — deliver
|
||||
each row to its scope (qm keychain / one-time secret drop), then delete
|
||||
the file.
|
||||
|
||||
## qm deployment directory
|
||||
|
||||
In the org's qm deployment repo (the directory `qm init` produced):
|
||||
|
||||
1. **Tool:** copy [`qm-harness-snippets/tool.json`](qm-harness-snippets/tool.json)
|
||||
to `sandbox/tools/gbrain/tool.json` and drop the compiled `gbrain` binary
|
||||
beside it (`bun build --compile --outfile gbrain src/cli.ts`, built for
|
||||
the sandbox image's OS/arch). `auth.credentialPaths` marks
|
||||
`~/.gbrain/config.json` as the scope's resident credential file;
|
||||
`auth.check` wires `gbrain whoami` into qm's connector status (read-scope;
|
||||
see the note below on why `remote doctor` cannot be used here).
|
||||
2. **Skill:** copy [`qm-harness-snippets/SKILL.md`](qm-harness-snippets/SKILL.md)
|
||||
to `sandbox/skills/gbrain/SKILL.md` (edit slug conventions to taste).
|
||||
3. Ship it: `qm sandbox build && qm sandbox publish && qm up`.
|
||||
|
||||
Per scope, one-time (agent- or operator-run, credentials from the handoff):
|
||||
|
||||
```bash
|
||||
gbrain init --mcp-only \
|
||||
--issuer-url https://brain.acme-example.com \
|
||||
--mcp-url https://brain.acme-example.com/mcp \
|
||||
--oauth-client-id gbrain_cl_... --oauth-client-secret gbrain_cs_...
|
||||
gbrain whoami # must succeed
|
||||
```
|
||||
|
||||
Use `--oauth-client-secret`, not `GBRAIN_REMOTE_CLIENT_SECRET`: an env-sourced
|
||||
secret is deliberately not written to `~/.gbrain/config.json`
|
||||
(`src/commands/init.ts`), so with the env var alone every later command fails
|
||||
once it leaves scope — and qm's `sandbox.secretEnv` is org-wide, so there is no
|
||||
per-scope env to keep it in. With the flag, the credential lands in the config
|
||||
file on the scope's durable disk and this runs once per scope, ever.
|
||||
|
||||
`gbrain remote doctor` is **not** the health check here: `run_doctor` is an
|
||||
`admin`-scope op and these clients are `read write` on purpose. `gbrain whoami`
|
||||
is read-scope and reports the client's identity, source, and grants.
|
||||
|
||||
## Verify isolation before rollout
|
||||
|
||||
From two differently-scoped sandboxes (or two thin-client configs):
|
||||
|
||||
```bash
|
||||
# alice-example (bound to emp-alice-example/, chan-eng/):
|
||||
gbrain put emp-alice-example/notes/test --content "mine" # OK
|
||||
gbrain put chan-eng/notes/test --content "shared" # OK
|
||||
gbrain put emp-bob-example/notes/test --content "not mine" # permission_denied
|
||||
gbrain put chan-product/notes/test --content "not my channel" # permission_denied
|
||||
gbrain search "test" # sees agents + org-wiki only
|
||||
```
|
||||
|
||||
## Cost + operations
|
||||
|
||||
- `search.mode balanced` (12K token budget, relational retrieval on) is the
|
||||
right default for a startup fleet; see `docs/guides/search-modes.md` for
|
||||
the cost matrix before changing it.
|
||||
- Budgets: `--budget-usd-per-day` is recorded on the client but only enforced
|
||||
on the `submit_agent` path (`src/core/minions/budget-meter.ts`), which these
|
||||
`read write` clients cannot reach — so it does **not** cap spend from
|
||||
ordinary `search`/`put_page` traffic. Treat runaway-agent containment as an
|
||||
open item: watch the admin SPA (`/admin`) and `gbrain search stats`, and cap
|
||||
at the model/harness layer.
|
||||
- Backfills on a live brain: `gbrain embed --stale --pace` (see Pace Mode in
|
||||
CLAUDE.md / `docs/operations/spend-controls.md`).
|
||||
|
||||
## Deliberately deferred
|
||||
|
||||
- **qm `MemoryService` decorator** (mirror notebook captures into gbrain,
|
||||
fan `recall` out and merge, `volunteer_context` push): needs a qm code
|
||||
change; today's integration is agent-initiated via the CLI + skill.
|
||||
- **MCP-native attach:** qm pins `strictMcpConfig` with only its in-process
|
||||
server, so gbrain's MCP-discovered brain-resident skillpacks don't reach
|
||||
qm agents; the sandbox skill above covers it.
|
||||
- **Read-side prefix fencing** (hard privacy for `emp-*/` inside a shared
|
||||
source) — tracked upstream; the roster layout is forward-compatible with
|
||||
it.
|
||||
+46
-2
@@ -79,13 +79,20 @@ to the HTTP server, so no migration is required.
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
On first start, the server prints an **admin bootstrap token** to stderr:
|
||||
On first start in an interactive terminal, the server prints an **admin
|
||||
bootstrap token** to stderr:
|
||||
|
||||
```
|
||||
Admin bootstrap token: 3a1f9c...
|
||||
Open http://localhost:3131/admin and paste it to log in.
|
||||
```
|
||||
|
||||
On a non-TTY start (systemd, Docker, any piped or captured logs) the generated
|
||||
token is hidden so it never lands in log storage. For headless deploys either
|
||||
set `GBRAIN_ADMIN_BOOTSTRAP_TOKEN` to a value you control before starting, or
|
||||
run `gbrain serve --http --print-admin-token` once on a trusted terminal to
|
||||
force printing.
|
||||
|
||||
Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
@@ -248,7 +255,7 @@ All 30 GBrain operations are available remotely, including `sync_brain` and
|
||||
directory where `gbrain serve` was launched. Symlinks, `..` traversal, and absolute
|
||||
paths outside cwd are rejected. Page slugs and filenames are allowlist-validated
|
||||
(alphanumeric + hyphens; no control chars, RTL overrides, or backslashes). Local
|
||||
CLI callers (`gbrain file upload ...`) keep unrestricted filesystem access since
|
||||
CLI callers (`gbrain files upload ...`) keep unrestricted filesystem access since
|
||||
the user owns the machine.
|
||||
|
||||
## Deployment Options
|
||||
@@ -256,6 +263,43 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# Conversation backfill durable outcomes
|
||||
|
||||
`gbrain extract-conversation-facts` stores page-level outcomes in `facts` so
|
||||
bulk runs, autopilot, and `gbrain doctor` can distinguish finished work from
|
||||
retryable work without adding another state table.
|
||||
|
||||
This is completion authority, not ordinary extracted knowledge. The authority
|
||||
is deliberately narrow: a marker is valid only for the exact page or transcript
|
||||
snapshot that was parsed, and only after every required operation succeeded.
|
||||
|
||||
## Outcome protocol
|
||||
|
||||
The current protocol is v2. Its source names are versioned so rows written by
|
||||
older best-effort implementations cannot suppress a corrective replay.
|
||||
|
||||
| Outcome | `facts.source` | Meaning |
|
||||
|---|---|---|
|
||||
| Complete | `cli:extract-conversation-facts:terminal:v2` | Every eligible segment was extracted and inserted successfully, the input remained unchanged, and the terminal write succeeded. |
|
||||
| Scanned, not extractable | `cli:extract-conversation-facts:non-extractable:v2` | A recognized input was scanned successfully but contained no eligible multi-message segment. |
|
||||
| Unfinished | no matching v2 outcome | Work is pending, failed, was not recognized, changed during extraction, or has only a legacy marker. |
|
||||
|
||||
The non-extractable outcome is intentionally separate from completion. It does
|
||||
not claim that knowledge facts were extracted. CLI counters, cycle details, and
|
||||
doctor output preserve that distinction.
|
||||
|
||||
## Snapshot identity
|
||||
|
||||
Every v2 marker binds `source_session` to the parser input snapshot:
|
||||
|
||||
```text
|
||||
<outcome-source>:<page-slug>:<version-token>
|
||||
```
|
||||
|
||||
There are two token forms.
|
||||
|
||||
### Database-backed page body
|
||||
|
||||
For pages parsed from `compiled_truth` and `timeline`, the token is:
|
||||
|
||||
```text
|
||||
page-<pages.content_hash>-<effective-date>
|
||||
```
|
||||
|
||||
`content_hash` covers title, type, compiled truth, timeline, and frontmatter.
|
||||
The effective-date suffix covers the remaining date input used by parsing. This
|
||||
identity does not depend on JavaScript's millisecond timestamp precision, so two
|
||||
writes within one PostgreSQL millisecond still produce different tokens when
|
||||
parser input changes. A legacy page with a null content hash uses a computed
|
||||
SHA-256 fallback and is verified in-process by both extraction and doctor.
|
||||
|
||||
### Raw transcript sidecar
|
||||
|
||||
When frontmatter contains `raw_transcript`, the source text lives outside the
|
||||
page row and may change without changing `pages.updated_at`. Its token is:
|
||||
|
||||
```text
|
||||
sidecar-<SHA-256>
|
||||
```
|
||||
|
||||
The digest covers the exact body given to the parser plus parser-relevant page
|
||||
metadata: title, type, frontmatter, and effective date. Selection recomputes
|
||||
the digest before skipping work. A sidecar-only edit therefore reopens the page.
|
||||
|
||||
`gbrain doctor` cannot read sidecars in its SQL aggregate, so it enumerates those
|
||||
pages in bounded batches and calls the same canonical verifier used by
|
||||
extraction. Doctor and extraction therefore agree after sidecar-only edits.
|
||||
|
||||
## Selection and locking
|
||||
|
||||
Bulk extraction follows this sequence:
|
||||
|
||||
1. Enumerate candidate pages in bounded batches.
|
||||
2. Filter candidates with matching v2 outcomes.
|
||||
3. Apply `--limit` to the remaining pages that actually need work.
|
||||
4. Acquire the source-and-slug advisory lock.
|
||||
5. Re-fetch the page under that lock.
|
||||
6. Recompute and recheck the snapshot-bound outcome.
|
||||
7. Prepare one immutable parser snapshot and process it.
|
||||
8. Re-fetch and recompute the snapshot before writing an outcome.
|
||||
|
||||
The pre-lock check avoids parser, filesystem, and model work for ordinary
|
||||
completed pages. The under-lock refetch prevents a stale enumeration object
|
||||
from becoming the certified input. The final comparison prevents an edit that
|
||||
happens during model or insertion work from receiving a marker for old content.
|
||||
|
||||
An edit can occur after the final comparison and before marker insertion. That
|
||||
is still safe because the marker contains the old version token. Future
|
||||
selection compares the token, not marker creation time, and reopens the page.
|
||||
|
||||
Single-page `--slug` runs use the same under-lock path.
|
||||
|
||||
## Strict extraction success
|
||||
|
||||
The general `extractFactsFromTurn` API remains best-effort for interactive
|
||||
callers. It historically returns an empty array for both a legitimate zero-fact
|
||||
answer and several model failures.
|
||||
|
||||
Conversation backfill instead uses `extractFactsFromTurnWithOutcome`, whose
|
||||
result separates:
|
||||
|
||||
- `{ ok: true, facts: [] }`, a successful extraction with no durable facts;
|
||||
- `{ ok: true, facts: [...] }`, a successful extraction with facts; and
|
||||
- `{ ok: false, reason, error? }`, an unavailable provider, provider error,
|
||||
refusal, content filter, malformed output, or repeated truncation.
|
||||
|
||||
Any failed segment aborts the page attempt. Any `insertFacts` failure also
|
||||
aborts it. The page receives neither a checkpoint advancement nor a terminal
|
||||
outcome. Facts inserted by earlier segments may remain temporarily, but the
|
||||
next claim deletes this command's rows for the page and replays cleanly.
|
||||
|
||||
Bulk workers continue past an individual page failure, but they do not hide it.
|
||||
`pages_failed` counts failed claims, stderr names each page, the CLI exits 1,
|
||||
the autopilot phase reports `warn`, and receipts/rollups classify the run as
|
||||
incomplete. A tolerant pool is therefore observable without sacrificing the
|
||||
rest of a large backfill.
|
||||
|
||||
This distinction is load-bearing. Treating a provider outage as a successful
|
||||
zero-fact response would make a transient failure durable and permanently hide
|
||||
the page from later runs.
|
||||
|
||||
## Non-extractable authority
|
||||
|
||||
A non-extractable marker is written only when all of the following are true:
|
||||
|
||||
- a deterministic or accepted parser format recognized the input;
|
||||
- ordinary segmentation produced no eligible multi-message segment;
|
||||
- the parser phase was not `no_match`;
|
||||
- cleanup of prior command-owned rows succeeded; and
|
||||
- the input snapshot was still current immediately before cleanup and write.
|
||||
|
||||
A `no_match` result stays unfinished so a new parser pattern, optional fallback,
|
||||
or corrected input can recover it. Oversize pages, disappeared pages, lock
|
||||
contention, dry runs, aborts, cleanup errors, provider failures, extraction
|
||||
failures, insertion failures, and outcome-write failures also stay unfinished.
|
||||
|
||||
Cleanup errors are never interpreted as "zero rows deleted." Propagating them
|
||||
prevents a fresh non-extractable marker from coexisting with stale extracted
|
||||
facts that could not be removed.
|
||||
|
||||
## Checkpoints are not authority
|
||||
|
||||
Operation checkpoints are only progress hints. They do not prove which page
|
||||
snapshot was processed, and old checkpoint entries do not include a snapshot
|
||||
token. When a page lacks a matching v2 outcome, the command discards that
|
||||
page's checkpoint entry and performs a delete-first full replay.
|
||||
|
||||
This rule prevents two corruption classes:
|
||||
|
||||
- edited text with timestamps older than the old watermark being skipped; and
|
||||
- command-owned facts being deleted while the checkpoint skips the segments
|
||||
needed to recreate them.
|
||||
|
||||
Deleting `op_checkpoints` does not reopen pages with matching v2 outcomes.
|
||||
Deleting or editing an outcome does not make a checkpoint authoritative.
|
||||
|
||||
## `--limit` semantics
|
||||
|
||||
`--limit N` caps pages that require processing, not completed pages inspected
|
||||
while finding them. Durable filtering happens before clipping a batch. With a
|
||||
completed page first and a pending page second, `--limit 1` processes the
|
||||
pending page rather than consuming the limit on the completed page.
|
||||
|
||||
`pages_considered` may therefore exceed `--limit` because it includes durable
|
||||
outcomes observed during selection. Model-bearing page work does not exceed the
|
||||
limit.
|
||||
|
||||
## `--force`
|
||||
|
||||
`--force` bypasses durable outcome selection and clears the page checkpoint.
|
||||
It still uses delete-first replay, strict extraction outcomes, advisory locks,
|
||||
and snapshot verification. Force means "recompute" rather than "relax safety."
|
||||
|
||||
## Operator signals
|
||||
|
||||
The result exposes separate counters:
|
||||
|
||||
- `pages_skipped_completed`
|
||||
- `pages_skipped_non_extractable`
|
||||
- `pages_marked_non_extractable`
|
||||
- `pages_failed`
|
||||
|
||||
The CLI aggregates these across sources. The autopilot backfill phase includes
|
||||
them in phase details. `gbrain doctor` reports `completed`,
|
||||
`scanned_not_extractable`, and `backlog` independently.
|
||||
|
||||
Run a small canary twice:
|
||||
|
||||
```bash
|
||||
gbrain extract-conversation-facts --source-id default --limit 10 --workers 1 --max-cost-usd 0.25 --yes
|
||||
gbrain extract-conversation-facts --source-id default --limit 10 --workers 1 --max-cost-usd 0.25 --yes
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
On the second run, unchanged pages should move through durable skip counters.
|
||||
Edit one page or raw transcript sidecar and rerun; that page should process
|
||||
again and receive a marker with a new token.
|
||||
|
||||
## Maintainer contracts
|
||||
|
||||
- Version completion protocols when their success guarantees change.
|
||||
- Require an exact `source`, page slug, and snapshot-bound `source_session`.
|
||||
- Keep completion and non-extractable as different sources and counters.
|
||||
- Re-fetch after acquiring the lock; never certify the enumeration object.
|
||||
- Revalidate the snapshot before writing either durable outcome.
|
||||
- Keep sidecar content in the version identity.
|
||||
- Keep regular-page content hash and effective date in the version identity.
|
||||
- Never turn model, insertion, cleanup, cancellation, or parser failures into
|
||||
successful empty extraction.
|
||||
- Never classify `no_match` or dry-run output as a durable negative.
|
||||
- Do not make operation checkpoints completion authority.
|
||||
- Apply work limits after durable filtering.
|
||||
- Keep doctor source-scoped by both page and fact `source_id`.
|
||||
- Give terminal completion precedence if both current outcome rows exist.
|
||||
- Update CLI and cycle aggregation whenever a result counter changes.
|
||||
|
||||
## Focused verification
|
||||
|
||||
```bash
|
||||
bun test test/extract-conversation-facts.test.ts
|
||||
bun test test/doctor-conversation-facts-backlog.test.ts
|
||||
bun x tsc --noEmit
|
||||
```
|
||||
|
||||
The focused suite covers checkpoint garbage collection, same-timestamp edits,
|
||||
edits during extraction, sidecar-only edits, legacy marker replay, provider and
|
||||
insert failures, cleanup failure, recognized non-extractable scans, retryable
|
||||
parser misses, post-filter limits, force replay, and doctor accounting.
|
||||
@@ -0,0 +1,240 @@
|
||||
# Conversation parser LLM fallback
|
||||
|
||||
The conversation parser has two stages:
|
||||
|
||||
1. A deterministic registry recognizes known transcript formats.
|
||||
2. An optional LLM fallback parses pages that every built-in pattern rejects.
|
||||
|
||||
The second stage is disabled by default. Enabling it is a privacy decision
|
||||
because unmatched transcript text can be sent to the configured utility-tier
|
||||
model provider.
|
||||
|
||||
## Enable or disable the fallback
|
||||
|
||||
Enable it for the current brain:
|
||||
|
||||
```bash
|
||||
gbrain config set conversation_parser.llm_fallback_enabled true
|
||||
```
|
||||
|
||||
Disable it:
|
||||
|
||||
```bash
|
||||
gbrain config set conversation_parser.llm_fallback_enabled false
|
||||
```
|
||||
|
||||
The key is registered explicitly, so neither command needs `--force`.
|
||||
Values other than the exact string `true` leave the fallback disabled.
|
||||
|
||||
The setting affects conversation fact extraction. It does not make the
|
||||
synchronous `conversation-parser scan` command call a model, and it does not
|
||||
enable the separate LLM polish scaffold.
|
||||
|
||||
## Select the utility model and run a canary
|
||||
|
||||
Inspect the model routing before enabling a production run:
|
||||
|
||||
```bash
|
||||
gbrain models
|
||||
```
|
||||
|
||||
The fallback uses the resolved `utility` tier. Override that tier when the
|
||||
brain should use a different configured provider or model:
|
||||
|
||||
```bash
|
||||
gbrain config set models.tier.utility <provider:model>
|
||||
```
|
||||
|
||||
Start with one known unmatched page and an explicit cost cap:
|
||||
|
||||
```bash
|
||||
gbrain extract-conversation-facts \
|
||||
--source-id <source-id> \
|
||||
--slug <conversation-slug> \
|
||||
--max-cost-usd 1
|
||||
```
|
||||
|
||||
Do not add `--dry-run` to this canary. Dry runs deliberately stop before the
|
||||
fallback boundary, so they cannot prove provider routing or model output.
|
||||
Success emits the per-page fallback log described under
|
||||
[Operator visibility](#operator-visibility). After the canary, remove `--slug`
|
||||
to process the source normally.
|
||||
|
||||
## When the fallback runs
|
||||
|
||||
For each eligible conversation page, extraction:
|
||||
|
||||
1. Reads the same body used by the deterministic parser, including a configured
|
||||
raw transcript sidecar for meeting pages.
|
||||
2. Calls `parseConversation(body, { page })`.
|
||||
3. Uses the deterministic messages when any built-in pattern succeeds.
|
||||
4. Calls the LLM fallback only when the parse phase is exactly `no_match`, the
|
||||
message list is empty, the opt-in key is `true`, and this is not a dry run.
|
||||
5. Splits accepted fallback messages into the normal extraction segments.
|
||||
|
||||
The fallback never replaces, edits, or polishes a successful deterministic
|
||||
parse. Adding a built-in pattern therefore removes model use for that format
|
||||
without changing configuration.
|
||||
|
||||
Dry runs remain local and cost-free. They report deterministic segmentation
|
||||
only and never send unmatched content to a provider.
|
||||
|
||||
## Data sent to the model
|
||||
|
||||
The full unmatched body is processed in overlapping windows of at most 100
|
||||
non-empty lines, with up to 20 lines of preceding context. Blank lines are
|
||||
omitted. Every model request receives:
|
||||
|
||||
- an instruction to treat the transcript as untrusted data;
|
||||
- an authoritative page date when one can be derived;
|
||||
- the sampled transcript inside an explicit chat-log envelope.
|
||||
|
||||
The system prompt tells the model not to follow commands or instructions found
|
||||
inside transcript content. It asks for message extraction only.
|
||||
|
||||
Each window is cached independently. Overlap results with the same normalized
|
||||
speaker and timestamp are deduplicated; when one body contains the other, the
|
||||
longer body wins. This preserves common multi-line messages that straddle a
|
||||
window boundary. If any later window has an ordinary provider or parse failure,
|
||||
the fallback returns no page result and extraction does not advance the
|
||||
checkpoint. Successful earlier windows stay cached for the retry.
|
||||
|
||||
Fallback calls allow up to 8,000 output tokens. Any non-terminal model stop,
|
||||
including length truncation, refusal, content filtering, tool use, or an
|
||||
unrecognized provider stop, is rejected before parsing and caching. A
|
||||
syntactically valid partial JSON array therefore cannot advance a checkpoint.
|
||||
|
||||
The utility model is resolved once per source run through the normal model
|
||||
configuration chain. The default fallback is the utility-tier Anthropic model.
|
||||
|
||||
## Date and timestamp behavior
|
||||
|
||||
The fallback uses the deterministic parser's date precedence:
|
||||
|
||||
1. an explicit caller date;
|
||||
2. `frontmatter.date`;
|
||||
3. the page effective date;
|
||||
4. `1970-01-01` when no date is known.
|
||||
|
||||
A real page date is included in both the prompt and the content-hash cache key.
|
||||
Two pages with identical time-only transcript text but different dates cannot
|
||||
share a cached parse.
|
||||
|
||||
Returned timestamps must be strict RFC3339 date-times with seconds and an
|
||||
explicit `Z` or numeric timezone offset. Calendar fields are validated before
|
||||
parsing. Accepted timestamps are normalized to whole-second UTC form:
|
||||
|
||||
```text
|
||||
YYYY-MM-DDTHH:MM:SSZ
|
||||
```
|
||||
|
||||
Date-only values, timezone-less values, impossible calendar dates, timestamps
|
||||
more than 24 hours in the future, blank speakers, and blank message bodies are
|
||||
discarded. Valid messages are stable-sorted by timestamp before segmentation.
|
||||
Canonical chronological UTC output keeps segment filtering and durable
|
||||
checkpoint comparisons stable and prevents future checkpoint poisoning.
|
||||
|
||||
If no page date is known, the prompt retains the historical epoch fallback.
|
||||
Full timestamps present in the transcript can still be extracted normally.
|
||||
|
||||
## Non-chat and failure behavior
|
||||
|
||||
The model is instructed to return an empty JSON array for non-chat content.
|
||||
An empty response, malformed JSON, unavailable provider, or transport failure
|
||||
leaves the page with no messages. Extraction skips that page and continues.
|
||||
|
||||
The fallback is fail-open with respect to parser availability. It does not turn
|
||||
a model outage into a deterministic-parser outage.
|
||||
|
||||
Cancellation and `BudgetExhausted` are control-flow signals, not provider
|
||||
failures. The extraction caller explicitly propagates them through the
|
||||
fail-open boundary so aborts stay prompt and hard cost caps remain effective.
|
||||
An `AbortError` from a provider timeout still fails open while the caller's own
|
||||
abort signal remains live.
|
||||
|
||||
The gateway can discover an underestimated budget overage only after the final
|
||||
provider result. Extraction checks tracker spend against its cap after the run,
|
||||
so an overage remains visible even when there is no next model reservation.
|
||||
|
||||
## Cache and repeat runs
|
||||
|
||||
Successful fallback results use the shared conversation-parser cache:
|
||||
|
||||
- an in-process map for repeat calls during one process;
|
||||
- the `conversation_parser_llm_cache` table for repeat calls across processes.
|
||||
|
||||
Each chunk's cache key includes the call shape, resolved model, page date
|
||||
metadata, and chunk content hash. A cached response is still validated before
|
||||
it originally enters the cache.
|
||||
|
||||
Once fallback messages produce extractable segments, the ordinary per-page
|
||||
checkpoint advances to the newest segment timestamp. A later run can read the
|
||||
cached parse, apply the checkpoint watermark, and skip already completed
|
||||
segments without another provider call.
|
||||
|
||||
## Operator visibility
|
||||
|
||||
`ExtractConversationFactsResult.pages_llm_fallback` counts pages for which the
|
||||
fallback returned at least one valid message. The command also logs:
|
||||
|
||||
```text
|
||||
[extract-conversation-facts] LLM fallback parsed N message(s) for <slug>
|
||||
```
|
||||
|
||||
The multi-source CLI summary reports the total number of fallback-parsed pages.
|
||||
A zero count means either the fallback was disabled, deterministic patterns
|
||||
handled every page, or fallback attempts returned no valid messages.
|
||||
|
||||
## Maintainer contracts
|
||||
|
||||
Keep these boundaries intact when changing the fallback:
|
||||
|
||||
- Default off. Page text must not reach the fallback without the exact opt-in.
|
||||
- Never call the provider during `--dry-run`.
|
||||
- Deterministic first. Invoke it only for phase `no_match`.
|
||||
- One model resolution per source run, not per page.
|
||||
- Use `deriveDateContext({ page })` so regex and LLM timestamps share metadata.
|
||||
- Put date metadata in the hashed request content to prevent cross-date cache
|
||||
collisions.
|
||||
- Process every non-empty line in bounded cached overlapping windows. Preserve
|
||||
common cross-boundary continuations through overlap and deterministic
|
||||
deduplication. Never checkpoint a partial page after a later window fails or
|
||||
returns a non-terminal stop reason.
|
||||
- Validate and canonicalize all model-produced fields before segmentation.
|
||||
- Stable-sort accepted messages before segmenting or checkpointing them.
|
||||
- Keep the exact config key in `KNOWN_CONFIG_KEYS`. Do not register the whole
|
||||
`conversation_parser.*` namespace while other scaffolded keys remain unwired.
|
||||
- Preserve `[]` and `null` as skip-page outcomes.
|
||||
- Propagate cancellation and budget-stop errors selected by the extraction
|
||||
caller; fail open only for ordinary provider and parse failures.
|
||||
- Never persist inferred regexes or promote model guesses into the built-in
|
||||
registry.
|
||||
|
||||
## Test coverage
|
||||
|
||||
The focused tests cover:
|
||||
|
||||
- default-off behavior with zero fallback calls;
|
||||
- enabled dry-run behavior with zero provider calls;
|
||||
- exact config-key registration;
|
||||
- a successful production-path fallback;
|
||||
- page-date prompt and cache-key separation;
|
||||
- durable checkpoint advancement and cache reuse;
|
||||
- complete processing beyond the first 100 non-empty lines;
|
||||
- cross-boundary continuation preservation and overlap deduplication;
|
||||
- rejection of truncated, refused, and content-filtered model results;
|
||||
- all-or-nothing page results when a later chunk fails;
|
||||
- non-chat empty arrays and malformed output;
|
||||
- strict timestamp normalization, ordering, and invalid-item filtering;
|
||||
- provider-unavailable and transport-failure behavior;
|
||||
- provider-timeout versus caller-cancellation behavior;
|
||||
- thrown and post-record budget-stop reporting.
|
||||
|
||||
Run the focused surface with:
|
||||
|
||||
```bash
|
||||
bun test test/conversation-parser/llm-base.test.ts \
|
||||
test/conversation-parser/llm-fallback.test.ts \
|
||||
test/extract-conversation-facts.test.ts \
|
||||
test/config-set.test.ts
|
||||
```
|
||||
@@ -49,6 +49,7 @@ The USD-limit knobs accept `off`, `unlimited`, or `none` (case-insensitive) to m
|
||||
| Backfill per-job budget | `embed.backfill_max_usd` | `10` | caps the job's tracker | `off` (`0` → default) | uncapped (still ledgered) |
|
||||
| Backfill cooldown | `embed.backfill_cooldown_min` | `10` | skips re-submission inside window | — (latency knob, not spend) | **not** bypassed |
|
||||
| `reindex-code` cost gate | — (preview before re-embed) | — | TTY prompt / non-TTY refuse + exit 2 | `--max-cost off` | informational |
|
||||
| `migrate embeddings` consent gate | — (plan + estimate before provider migration) | — | TTY y/N prompt / non-TTY refuse + exit 2 | `--yes` | estimate marked informational, but **still prompts** (guards a destructive schema rebuild, not just spend) |
|
||||
| `enrich` / `onboard --auto` | `--max-usd` (per-call) | — | refuse without a cap (non-TTY) | `--max-usd off` | runs uncapped (still ledgered) |
|
||||
|
||||
### Sync inline-embed cost gate
|
||||
|
||||
@@ -140,6 +140,9 @@ Stable phase names shipped in v0.15.2:
|
||||
- `import.files`
|
||||
- `sync.deletes`, `sync.renames`, `sync.imports`
|
||||
- `migrate.copy_pages`, `migrate.copy_links`
|
||||
- `migrate.reembed` (the re-embed pass of `gbrain migrate embeddings`; total is the
|
||||
stale-chunk backlog at the start of the pass, so it can grow slightly if a
|
||||
writer adds chunks mid-run)
|
||||
- `repair_jsonb.run`, `repair_jsonb.<table>.<column>`
|
||||
- `backlinks.scan`
|
||||
- `lint.pages`
|
||||
|
||||
@@ -51,6 +51,18 @@ When storage configuration is present, `gbrain sync` automatically manages `.git
|
||||
- Skipped when the repo is a git submodule (`.git` is a file, not a directory) — submodule .gitignore changes don't survive parent updates. A warning explains.
|
||||
- Skipped entirely when `GBRAIN_NO_GITIGNORE=1` is set (escape hatch for shared-repo setups where a maintainer wants gbrain to leave .gitignore alone).
|
||||
- Failures (write permission denied, etc.) are caught and logged, never crash sync.
|
||||
- Warns when a configured collector's declared output dir (recipe `output_paths`
|
||||
frontmatter) sits inside a `db_only` path: gitignored files never appear in the
|
||||
git-walking sync diff, and `gbrain import` honors `.gitignore` too — the
|
||||
collector would run green while nothing reaches the DB. The
|
||||
`db_only_collector_collision` doctor check surfaces the same trap.
|
||||
|
||||
Related doctor coverage: `undeclared_db_only_pages` warns about DB pages with no
|
||||
backing file that sit outside every declared `db_only` path. The engine's own
|
||||
derive-phase output prefixes (`life/events/`, `atoms/`, `extracts/`,
|
||||
`dream-cycle-summaries/`) count as implicitly declared for that check, so healthy
|
||||
brains stay quiet without adding them to `gbrain.yml`. They are NOT auto-added to
|
||||
`.gitignore` — only explicitly declared `db_only` dirs are.
|
||||
|
||||
Example `.gitignore` addition:
|
||||
|
||||
|
||||
@@ -0,0 +1,690 @@
|
||||
# Engine Dynamic-Import Reconciliation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Reconstruct the missing engine-path static-import hardening, preserve the four load-bearing lazy gateway fallbacks, and prevent unreviewed dynamic imports from returning.
|
||||
|
||||
**Architecture:** Make the 13 safe engine/migration import statements static and leave only four line-marked `ai/gateway.ts` imports inside their existing soft-failure `try/catch` boundaries. Enforce that current state with a repository-anchored Bash wrapper delegating to a fail-closed TypeScript AST scanner, a hermetic Bun regression test, package/verify wiring, and current-state architecture documentation.
|
||||
|
||||
**Tech Stack:** TypeScript compiler API, Bun test runner, Bash, Git, generated llms documentation bundles.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Reconstruct directly on branch `claude/kind-meitner-330c90`, based on investigated `origin/master` commit `6136e139972a5449630b4f47f5ed7b4cbe5b811b` plus design commit `d7f52d8c`.
|
||||
- Do not merge or cherry-pick `48ada48f`, `248bfe55`, `ef4cf7a8`, or either historical branch wholesale.
|
||||
- Do not modify `VERSION`, `CHANGELOG.md`, `TODOS.md`, or release metadata; this is a no-version-bump reconciliation.
|
||||
- Keep all four `await import('./ai/gateway.ts')` calls lazy: PGLite and Postgres `initSchema`, plus both `_upsertChunksOnce` methods.
|
||||
- Every allowed lazy gateway line must carry `engine-dynamic-import-ok`; there is no file-level exemption.
|
||||
- Preserve the stronger gateway rationale: the static closure is large, and eager module evaluation would occur outside the local `try/catch`, potentially converting a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
- Describe the hoists as engine-path hardening. Do not claim every dynamic import deterministically causes a Windows crash; system-wide commit exhaustion confounded prior measurements.
|
||||
- Keep shared PGLite/Postgres behavior in parity.
|
||||
- Invoke repository shell scripts through `bash` in `package.json`.
|
||||
- Capture complete test/check output to workspace-local `.context/*.txt` files before inspecting it; never pipe a test command directly through `head` or `tail`.
|
||||
- Use `git log -G`, not `git log -S`, for any additional dynamic-to-static import history work.
|
||||
- Keep every implementation and verification commit local. Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after local completion.
|
||||
- Before editing any affected function, run GBrain `code_blast` and `code_callers` for that symbol and inspect any disambiguation candidates.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
- Create `scripts/check-engine-dynamic-import.sh` — repository-anchored Bash wrapper for default and explicit input routing.
|
||||
- Create `scripts/check-engine-dynamic-import.ts` — TypeScript AST policy scanner for runtime `import()` expressions, parse/read failures, and exact-line comment-trivia opt-outs.
|
||||
- Create `test/scripts/check-engine-dynamic-import.test.ts` — 22 hermetic adversarial, CRLF, fail-closed, real-tree, and wiring tests.
|
||||
- Modify `src/core/pglite-engine.ts` — hoist three safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/postgres-engine.ts` — hoist eight safe import statements and mark two deliberate gateway imports.
|
||||
- Modify `src/core/migrate.ts` — hoist two safe migration helper import statements.
|
||||
- Modify `package.json` — expose `check:engine-dynamic-import` and append it to `check:all` through `bash`.
|
||||
- Modify `scripts/run-verify-parallel.sh` — add the package check to the authoritative verify dispatcher.
|
||||
- Modify `CLAUDE.md` — add the cross-cutting current-state invariant.
|
||||
- Modify `docs/architecture/KEY_FILES.md` — update current-state entries for the three engine-path files.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` — required derived bundles after CLAUDE/reference documentation changes.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Establish and enforce the source invariant
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/check-engine-dynamic-import.sh`
|
||||
- Create: `scripts/check-engine-dynamic-import.ts`
|
||||
- Create: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `src/core/pglite-engine.ts`
|
||||
- Modify: `src/core/postgres-engine.ts`
|
||||
- Modify: `src/core/migrate.ts`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: shell positional arguments `FILE...`; without arguments, the guard scans the three repository files.
|
||||
- Produces: `scripts/check-engine-dynamic-import.sh [FILE...]`, exit `0` when every runtime dynamic import is allowed and exit `1` after reporting every `file:line:text` violation plus every read/parse error on stderr.
|
||||
- Produces: one line-level opt-out token, `engine-dynamic-import-ok`, accepted only in real comment trivia on the same physical line as the deliberately lazy import.
|
||||
- Fails closed on missing/unreadable inputs, TypeScript parse diagnostics, and scanner/process failures; comments, strings, templates, regex literals, and type-position `import(...)` syntax are not runtime imports.
|
||||
|
||||
- [ ] **Step 1: Record call-graph blast radius before touching functions**
|
||||
|
||||
First call `sources_list` and select the source whose registered path is this gbrain checkout. Then run `code_blast` and `code_callers` for these qualified symbols with that exact `source_id`, following `did_you_mean`/`candidates` when a method name is ambiguous:
|
||||
|
||||
```text
|
||||
src/core/pglite-engine.ts::PGLiteEngine.initSchema
|
||||
src/core/pglite-engine.ts::PGLiteEngine.batchRetry
|
||||
src/core/pglite-engine.ts::PGLiteEngine._upsertChunksOnce
|
||||
src/core/pglite-engine.ts::PGLiteEngine.mergeOntologyFact
|
||||
src/core/pglite-engine.ts::PGLiteEngine.getRecentSalience
|
||||
src/core/postgres-engine.ts::PostgresEngine.disconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.initSchema
|
||||
src/core/postgres-engine.ts::PostgresEngine.batchRetry
|
||||
src/core/postgres-engine.ts::PostgresEngine._upsertChunksOnce
|
||||
src/core/postgres-engine.ts::PostgresEngine.mergeOntologyFact
|
||||
src/core/postgres-engine.ts::PostgresEngine.reconnect
|
||||
src/core/postgres-engine.ts::PostgresEngine.getRecentSalience
|
||||
src/core/migrate.ts::runMigrationSQLWithRetry
|
||||
src/core/migrate.ts::runMigrations
|
||||
```
|
||||
|
||||
Use `depth: 5`, `max_nodes: 200`, and `limit: 100`. Expected: no caller requires a signature or behavior change; the patch only changes module binding time and retains all local fallback/error handling.
|
||||
|
||||
- [ ] **Step 2: Write the failing guard regression test**
|
||||
|
||||
Create `test/scripts/check-engine-dynamic-import.test.ts` as a hermetic subprocess suite. The completed 22-test surface covers:
|
||||
|
||||
- unmarked runtime `import()` rejection, including bare and trivia-separated forms;
|
||||
- same-line markers in real line or multiline block-comment trivia;
|
||||
- rejection of markers on prior lines or inside strings, templates, and module paths;
|
||||
- comments and comment-like delimiters inside strings, templates, and regex literals;
|
||||
- live code after same-line or multiline block comments close;
|
||||
- CRLF input and complete multi-file violation aggregation;
|
||||
- missing/readable mixed inputs and TypeScript parse diagnostics;
|
||||
- default repository anchoring when invoked from a foreign Git repository;
|
||||
- the reconciled three-file source scan plus package/parallel-verifier wiring.
|
||||
|
||||
Use the TypeScript parser rather than a partial lexical reimplementation. On Windows, set the test default to 30 seconds because each case launches Git Bash and Bun, whose startup can exceed Bun's 5-second per-test default.
|
||||
|
||||
- [ ] **Step 3: Run the test to prove the pre-implementation red state**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result captured inside the log. At minimum, the `exists` assertion fails because `scripts/check-engine-dynamic-import.sh` does not exist. Read `.context/engine-dynamic-import-red.txt`; do not infer the result from a truncated pipeline.
|
||||
|
||||
- [ ] **Step 4: Add the CRLF-safe, fail-closed guard**
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.sh` as a thin LF-terminated wrapper. Resolve its own directory first; when no explicit files are passed, anchor the repository with `git -C "$SCRIPT_DIR/.."` and scan the two engines plus `migrate.ts`. Delegate with `exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"` so scanner failures propagate.
|
||||
|
||||
Create `scripts/check-engine-dynamic-import.ts` using the TypeScript compiler API:
|
||||
|
||||
- read every requested file and aggregate read failures;
|
||||
- parse as TypeScript and aggregate parse diagnostics;
|
||||
- walk the AST for `CallExpression`s whose expression is `ImportKeyword`;
|
||||
- locate all marker occurrences in the full source and use `ts.getTokenAtPosition` to admit only occurrences outside AST tokens (real comment trivia), recording their physical source lines;
|
||||
- require each runtime import's line to have an admitted marker or report its original `file:line:text`;
|
||||
- print every read/parse error and every violation before exiting nonzero.
|
||||
|
||||
This preserves CRLF line accounting, ignores comment/literal/type-only false positives, catches every legal runtime `import()` shape the TypeScript parser recognizes, rejects marker spoofing, and fails closed.
|
||||
|
||||
- [ ] **Step 5: Run the guard test to prove the source-tree midpoint is still red**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-midpoint.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: the synthetic violation, marker, comments, and CRLF cases pass. The default repository scan fails and reports all 17 current imports: 13 unmarked safe candidates plus the four not-yet-marked gateway calls.
|
||||
|
||||
- [ ] **Step 6: Hoist the three safe PGLite import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add the ontology/recency imports near the top of `src/core/pglite-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
```
|
||||
|
||||
Delete only these three in-method destructuring imports, leaving their uses unchanged:
|
||||
|
||||
```ts
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Mark both PGLite gateway soft-failure boundaries**
|
||||
|
||||
In `PGLiteEngine.initSchema`, preserve the `try/catch` and accessors, changing only the rationale and import line:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PGLiteEngine._upsertChunksOnce`, preserve the config-row and compile-time fallback chain:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Hoist the eight safe Postgres import statements**
|
||||
|
||||
Replace the existing `retry.ts` import and add these imports near the top of `src/core/postgres-engine.ts`:
|
||||
|
||||
```ts
|
||||
// Engine-path imports stay static unless a call site carries an explicit
|
||||
// engine-dynamic-import-ok justification. The gateway is the only current
|
||||
// exception because its local try/catch preserves a soft fallback.
|
||||
import {
|
||||
withRetry,
|
||||
BULK_RETRY_OPTS,
|
||||
resolveBulkRetryOpts,
|
||||
computeNextDelay,
|
||||
isRetryableConnError,
|
||||
type BatchAuditSite,
|
||||
} from './retry.ts';
|
||||
import { isConnectionEndedError } from './retry-matcher.ts';
|
||||
import {
|
||||
valueHash,
|
||||
normalizeDimension,
|
||||
isNovelDimension,
|
||||
} from './chronicle/ontology.ts';
|
||||
import {
|
||||
resolveRecencyDecayMap,
|
||||
DEFAULT_FALLBACK,
|
||||
} from './search/recency-decay.ts';
|
||||
import { logDbDisconnect } from './audit/db-disconnect-audit.ts';
|
||||
import { logPoolRecovery } from './audit/pool-recovery-audit.ts';
|
||||
```
|
||||
|
||||
Delete the eight safe dynamic-import statements while keeping their surrounding `try/catch` blocks and calls unchanged:
|
||||
|
||||
```ts
|
||||
const { logDbDisconnect } = await import('./audit/db-disconnect-audit.ts');
|
||||
const { isRetryableConnError } = await import('./retry.ts');
|
||||
const { valueHash, normalizeDimension, isNovelDimension } = await import('./chronicle/ontology.ts');
|
||||
const { isConnectionEndedError } = await import('./retry-matcher.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { logPoolRecovery } = await import('./audit/pool-recovery-audit.ts');
|
||||
const { resolveRecencyDecayMap, DEFAULT_FALLBACK } = await import('./search/recency-decay.ts');
|
||||
```
|
||||
|
||||
Update the stale `batchRetry` comment from “Lazy-import to avoid a circular dep concern” to current truth:
|
||||
|
||||
```ts
|
||||
// retry.ts is already in this module's static graph through withRetry, so
|
||||
// classifying the exhausted error does not need a second runtime import.
|
||||
```
|
||||
|
||||
- [ ] **Step 9: Mark both Postgres gateway soft-failure boundaries**
|
||||
|
||||
In `PostgresEngine.initSchema`, mirror the PGLite rationale and preserve behavior:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy: its static closure is large, and evaluation inside
|
||||
// this try/catch preserves the unconfigured-gateway default fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
```
|
||||
|
||||
In `PostgresEngine._upsertChunksOnce`, preserve the DB-config fallback:
|
||||
|
||||
```ts
|
||||
try {
|
||||
// Keep the gateway lazy so module-load failure remains inside this soft
|
||||
// fallback boundary; eager evaluation would bypass the config-row fallback.
|
||||
const gw = await import('./ai/gateway.ts'); // engine-dynamic-import-ok
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
```
|
||||
|
||||
- [ ] **Step 10: Hoist the two migration helper import statements**
|
||||
|
||||
Add these static imports at the top of `src/core/migrate.ts`:
|
||||
|
||||
```ts
|
||||
// runMigrations executes while an initialized engine is live. Keep its helper
|
||||
// modules in the static graph rather than importing them from async handlers.
|
||||
import {
|
||||
isStatementTimeoutError,
|
||||
isRetryableConnError,
|
||||
} from './retry-matcher.ts';
|
||||
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
|
||||
```
|
||||
|
||||
Delete only these two local destructuring imports:
|
||||
|
||||
```ts
|
||||
const { isStatementTimeoutError, isRetryableConnError } = await import('./retry-matcher.ts');
|
||||
const { repairTimelineDedupIndex } = await import('./timeline-dedup-repair.ts');
|
||||
```
|
||||
|
||||
- [ ] **Step 11: Run the complete guard test and direct guard**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; output contains `check-engine-dynamic-import: ok (3 file(s) scanned)`.
|
||||
|
||||
- [ ] **Step 12: Prove the guard leaves exactly four marked dynamic imports**
|
||||
|
||||
```bash
|
||||
git grep -n -F "import('./ai/gateway.ts'); // engine-dynamic-import-ok" -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts > .context/engine-dynamic-import-sites.txt; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exactly four lines, all importing `./ai/gateway.ts` and all carrying `engine-dynamic-import-ok`; no match in `src/core/migrate.ts`.
|
||||
|
||||
- [ ] **Step 13: Run focused behavior tests**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. If Windows resource pressure aborts the process, record the exact exit code and rerun the failing file alone; do not relabel an infrastructure abort as a source pass.
|
||||
|
||||
- [ ] **Step 14: Commit the source invariant locally**
|
||||
|
||||
```bash
|
||||
git add scripts/check-engine-dynamic-import.sh scripts/check-engine-dynamic-import.ts test/scripts/check-engine-dynamic-import.test.ts src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "fix(engine): reconcile dynamic import hardening"
|
||||
```
|
||||
|
||||
Expected: one local commit; no version or release files staged.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Wire the guard into repository checks
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/scripts/check-engine-dynamic-import.test.ts`
|
||||
- Modify: `package.json`
|
||||
- Modify: `scripts/run-verify-parallel.sh`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `scripts/check-engine-dynamic-import.sh` from Task 1.
|
||||
- Produces: package script `check:engine-dynamic-import` and verify dry-list entry of the same name.
|
||||
|
||||
- [ ] **Step 1: Add failing wiring assertions**
|
||||
|
||||
Add these imports/constants to `test/scripts/check-engine-dynamic-import.test.ts`:
|
||||
|
||||
```ts
|
||||
const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json');
|
||||
```
|
||||
|
||||
Append this test block:
|
||||
|
||||
```ts
|
||||
describe('engine dynamic-import guard wiring', () => {
|
||||
it('is invoked through bash by check:all', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf8')) as {
|
||||
scripts: Record<string, string>;
|
||||
};
|
||||
expect(pkg.scripts['check:engine-dynamic-import']).toBe(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
expect(pkg.scripts['check:all']).toContain(
|
||||
'bash scripts/check-engine-dynamic-import.sh',
|
||||
);
|
||||
});
|
||||
|
||||
it('is listed by the authoritative verify dispatcher', () => {
|
||||
const result = spawnSync(BASH, [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(new Set((result.stdout ?? '').trim().split('\n'))).toContain(
|
||||
'check:engine-dynamic-import',
|
||||
);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the test and verify both wiring assertions fail**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-red.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit 0
|
||||
```
|
||||
|
||||
Expected: non-zero Bun result. The source guard tests remain green; package-script and verify-list assertions fail because the wiring is absent.
|
||||
|
||||
- [ ] **Step 3: Add the package scripts**
|
||||
|
||||
In `package.json`, add this script alongside the other `check:*` entries:
|
||||
|
||||
```json
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh"
|
||||
```
|
||||
|
||||
Append the guard to the existing `check:all` chain, preserving every existing check:
|
||||
|
||||
```text
|
||||
&& bash scripts/check-engine-dynamic-import.sh
|
||||
```
|
||||
|
||||
Do not rewrite any existing shell entry without its `bash` prefix.
|
||||
|
||||
- [ ] **Step 4: Add the authoritative verify entry**
|
||||
|
||||
In `scripts/run-verify-parallel.sh`, add this stable `CHECKS` entry near the other source-shape guards:
|
||||
|
||||
```bash
|
||||
"check:engine-dynamic-import"
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run the regression test and package check**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-wiring-green.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bun run check:engine-dynamic-import > .context/engine-dynamic-import-package-check.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0` and three files scanned.
|
||||
|
||||
- [ ] **Step 6: Commit the wiring locally**
|
||||
|
||||
```bash
|
||||
git add package.json scripts/run-verify-parallel.sh test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "test(engine): guard dynamic import policy"
|
||||
```
|
||||
|
||||
Expected: one local commit with the guard wiring and its regression assertions.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Document the current-state invariant
|
||||
|
||||
**Files:**
|
||||
- Modify: `CLAUDE.md`
|
||||
- Modify: `docs/architecture/KEY_FILES.md`
|
||||
- Regenerate: `llms.txt`
|
||||
- Regenerate: `llms-full.txt`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the four-marked-import source state and the `check:engine-dynamic-import` package surface.
|
||||
- Produces: current-state contributor guidance and fresh generated documentation bundles.
|
||||
|
||||
- [ ] **Step 1: Add the cross-cutting invariant to `CLAUDE.md`**
|
||||
|
||||
Add this bullet under “Cross-cutting invariants” near the other language/filesystem guards:
|
||||
|
||||
```md
|
||||
- **Engine-live paths use static imports by default.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, helper modules are top-level imports. The only current
|
||||
exceptions are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
```
|
||||
|
||||
Do not add release tags, Windows-crash certainty, or historical branch names.
|
||||
|
||||
- [ ] **Step 2: Update the PGLite current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this current-state sentence to the existing `src/core/pglite-engine.ts` entry, preserving the entry as one bullet:
|
||||
|
||||
```md
|
||||
Engine-path helper dependencies (`retry`, ontology, recency decay) bind statically; the only lazy imports are `ai/gateway.ts` in `initSchema` and `_upsertChunksOnce`, line-marked because their local catches preserve compiled-default and stored-config fallbacks that eager module evaluation would bypass.
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update the Postgres current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the existing `src/core/postgres-engine.ts` entry:
|
||||
|
||||
```md
|
||||
Retry classifiers, ontology/recency helpers, and disconnect/pool-recovery audit writers bind statically; only the two `ai/gateway.ts` fallback lookups stay lazy and line-marked, in parity with PGLite.
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update the migration current-state entry in `KEY_FILES.md`**
|
||||
|
||||
Append this sentence to the canonical `src/core/migrate.ts` entry (the broad runner entry, not the older v95-specific index note):
|
||||
|
||||
```md
|
||||
`retry-matcher.ts` and `timeline-dedup-repair.ts` are static dependencies because `runMigrations()` executes from live engine initialization; the engine dynamic-import guard scans this file with both engine implementations.
|
||||
```
|
||||
|
||||
Keep all three entries current-state only: no `v0.42.x`, branch, commit, “previously,” or “was/now” narration.
|
||||
|
||||
- [ ] **Step 5: Regenerate the llms bundles**
|
||||
|
||||
```bash
|
||||
bun run build:llms > .context/engine-dynamic-import-build-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; `llms.txt` and/or `llms-full.txt` update according to their configured linked/inlined status. Byte-identical output for a linked source is acceptable; the freshness test is authoritative.
|
||||
|
||||
- [ ] **Step 6: Run documentation freshness checks**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-llms-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
```bash
|
||||
bun run check:doc-history > .context/engine-dynamic-import-doc-history.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; no release-history marker is introduced into current-state reference docs.
|
||||
|
||||
- [ ] **Step 7: Confirm prohibited release files remain untouched**
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD -- VERSION CHANGELOG.md TODOS.md
|
||||
```
|
||||
|
||||
Expected: no output.
|
||||
|
||||
- [ ] **Step 8: Commit documentation and generated bundles locally**
|
||||
|
||||
```bash
|
||||
git add CLAUDE.md docs/architecture/KEY_FILES.md llms.txt llms-full.txt
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs(engine): record static import invariant"
|
||||
```
|
||||
|
||||
Expected: one local documentation commit. If one generated bundle is byte-identical, Git simply omits it.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Verify and review the complete local reconciliation
|
||||
|
||||
**Files:**
|
||||
- Verify all files changed since `d7f52d8c`.
|
||||
- Do not create or modify release/publication metadata.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Tasks 1–3.
|
||||
- Produces: full local verification evidence and an implementation diff ready for user review, not publication.
|
||||
|
||||
- [ ] **Step 1: Run the regression test and direct guard again**
|
||||
|
||||
```bash
|
||||
bun test test/scripts/check-engine-dynamic-import.test.ts > .context/engine-dynamic-import-final-test.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; the full guard regression suite passes.
|
||||
|
||||
```bash
|
||||
bash scripts/check-engine-dynamic-import.sh > .context/engine-dynamic-import-final-guard.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; three files scanned.
|
||||
|
||||
- [ ] **Step 2: Run TypeScript checking**
|
||||
|
||||
```bash
|
||||
bun run typecheck > .context/engine-dynamic-import-typecheck.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`. Report exact diagnostics if the branch or current Windows environment has a pre-existing failure.
|
||||
|
||||
- [ ] **Step 3: Run the authoritative verify dispatcher**
|
||||
|
||||
```bash
|
||||
bun run verify > .context/engine-dynamic-import-verify.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`, including `check:engine-dynamic-import`. On Windows, classify any per-check timeout from the complete log instead of treating the aggregate result as a source regression without evidence.
|
||||
|
||||
- [ ] **Step 4: Re-run focused tests as an ownership check**
|
||||
|
||||
```bash
|
||||
bun test test/chronicle-ontology.test.ts test/chronicle-ontology-ops.test.ts test/recency-decay.test.ts test/core/retry.test.ts test/retry-matcher.test.ts test/audit/pool-recovery-audit.test.ts test/migrate-retry.test.ts test/timeline-dedup-repair.test.ts > .context/engine-dynamic-import-final-focused.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`; record any infrastructure abort separately and rerun only the named file before classifying it.
|
||||
|
||||
- [ ] **Step 5: Run the llms freshness test after all documentation settles**
|
||||
|
||||
```bash
|
||||
bun test test/build-llms.test.ts > .context/engine-dynamic-import-final-llms.txt 2>&1; rc=$?; printf 'EXIT=%s\n' "$rc"; exit "$rc"
|
||||
```
|
||||
|
||||
Expected: exit `0`.
|
||||
|
||||
- [ ] **Step 6: Run whitespace and scope checks**
|
||||
|
||||
```bash
|
||||
git diff --check d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected: exit `0`, no output.
|
||||
|
||||
```bash
|
||||
git diff --name-only d7f52d8c..HEAD
|
||||
```
|
||||
|
||||
Expected files only:
|
||||
|
||||
```text
|
||||
CLAUDE.md
|
||||
docs/architecture/KEY_FILES.md
|
||||
docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
llms-full.txt
|
||||
llms.txt
|
||||
package.json
|
||||
scripts/check-engine-dynamic-import.sh
|
||||
scripts/check-engine-dynamic-import.ts
|
||||
scripts/run-verify-parallel.sh
|
||||
src/core/migrate.ts
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
test/scripts/check-engine-dynamic-import.test.ts
|
||||
```
|
||||
|
||||
Either generated llms file may be absent if regeneration proves it byte-identical. `VERSION`, `CHANGELOG.md`, and `TODOS.md` must be absent.
|
||||
|
||||
- [ ] **Step 7: Review the exact implementation diff**
|
||||
|
||||
```bash
|
||||
git diff --stat d7f52d8c..HEAD && git diff d7f52d8c..HEAD -- src/core/pglite-engine.ts src/core/postgres-engine.ts src/core/migrate.ts scripts/check-engine-dynamic-import.sh test/scripts/check-engine-dynamic-import.test.ts package.json scripts/run-verify-parallel.sh CLAUDE.md docs/architecture/KEY_FILES.md
|
||||
```
|
||||
|
||||
Expected review findings:
|
||||
|
||||
- Exactly 13 safe `await import(...)` statements are removed.
|
||||
- Exactly four `ai/gateway.ts` imports remain, all marked on the same line.
|
||||
- All four gateway imports remain inside their original local `try/catch` fallback boundaries.
|
||||
- No accessor logic, fallback ordering, SQL, public signature, or engine parity behavior changes.
|
||||
- The parser-backed guard reports all violations plus read/parse failures, preserves CRLF line accounting, ignores comments/literals/type-only syntax, detects every runtime `import()` call expression, and accepts opt-outs only from real comment trivia on the same physical line.
|
||||
- The package script invokes the shell guard through Bash; `check:all` invokes that shell guard directly, and the parallel verify dispatcher invokes the package check.
|
||||
- Documentation is current-state and makes no deterministic Windows-crash claim.
|
||||
|
||||
**Observed Windows verification classification:** The authoritative aggregate completed with 25 of 33 checks passing. Individual reruns showed `check:test-names` and `typecheck` green; privacy/isolation exceeded Windows timing budgets; WASM failed in unrelated temporary-symlink setup; eval-glossary was CRLF/LF drift; resolver/brain-first findings predated and did not intersect this branch. The focused aggregate produced 103 pass / 5 fail: three setup-hook timeouts reproduced at the untouched base, and the known `migrate-retry` polling failure reproduced there. Its additional race-status assertion did not reproduce at base, so it remains an unresolved timing-sensitive limitation in untouched code—not evidence of an in-scope defect and not claimed as conclusively pre-existing.
|
||||
|
||||
- [ ] **Step 8: Commit the approved plan document locally**
|
||||
|
||||
The plan is an approved, tracked execution artifact and must not be left as an uncommitted file after implementation:
|
||||
|
||||
```bash
|
||||
git add docs/superpowers/plans/2026-07-28-engine-dynamic-import-reconciliation.md
|
||||
```
|
||||
|
||||
```bash
|
||||
git commit -m "docs: plan engine dynamic-import reconciliation"
|
||||
```
|
||||
|
||||
Expected: one local plan commit; no release metadata staged.
|
||||
|
||||
- [ ] **Step 9: Inspect final status without publishing**
|
||||
|
||||
```bash
|
||||
git status --short --branch
|
||||
```
|
||||
|
||||
Expected: branch `claude/kind-meitner-330c90` with a clean working tree. No push, PR, upstream comment, or other external side effect.
|
||||
|
||||
- [ ] **Step 10: Capture the completed milestone to memory**
|
||||
|
||||
Before writing, search MemPalace wing `gbrain` for this exact reconciliation to avoid duplication. Add a verbatim drawer recording exact base/head commits, the 13 hoists, four gateway opt-outs and rationale, guard/test/docs files, every verification command with exit code, and any environment-owned failures. Add a GBrain project timeline entry only if there is an existing relevant gbrain project page; do not create duplicate release metadata.
|
||||
|
||||
- [ ] **Step 11: Report the local result and ask separately before publication**
|
||||
|
||||
Report:
|
||||
|
||||
- exact local commits;
|
||||
- changed files;
|
||||
- test/check exit codes;
|
||||
- any blocked or pre-existing failures;
|
||||
- confirmation that release files were untouched;
|
||||
- confirmation that nothing was pushed or published.
|
||||
|
||||
Do not run any publication command. Wait for explicit user approval before any push, PR, or upstream interaction.
|
||||
@@ -0,0 +1,175 @@
|
||||
# Scalar-source Backlink Validation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make backlink validation compare exact `(source_id, slug)` endpoint identities while preserving existing scalar, unscoped, and federated link-read semantics.
|
||||
|
||||
**Architecture:** Enrich every engine link-read row with the source identity of its joined from, to, and visible origin pages. Pass the validated page's scalar or federated scope into validator context; the backlink validator scopes its initial read consistently, groups targets by exact identity, and accepts only an exact reverse endpoint pair. SQL predicates remain unchanged, so trusted scalar cross-source visibility and federated all-endpoint containment remain intact.
|
||||
|
||||
**Tech Stack:** TypeScript, Bun test, PGLite, PostgreSQL/postgres.js.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Use strict red-before-green TDD with duplicate slugs across sources.
|
||||
- Preserve unscoped historical reads, scalar near-endpoint scoping, scalar explicit cross-source visibility, federated all-endpoint containment, and `sourceIds` precedence.
|
||||
- Keep PostgreSQL and PGLite projections in parity.
|
||||
- Do not change schema or conditional-write conflict semantics.
|
||||
- Keep deployment, restart, migration, and push actions outside the implementation tasks; a separately authorized release workflow may perform them after verification.
|
||||
- Capture full test output to files before inspecting it.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Pin the backlink false-negative in PGLite
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/writer.test.ts`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backLinkValidator.validate(PageValidationContext)` and source-qualified `putPage`/`addLink`.
|
||||
- Produces: regressions for wrong-source reverse rejection, exact reverse acceptance, cross-source pair acceptance, and exact target deduplication.
|
||||
|
||||
- [ ] **Step 1: Add the minimal failing duplicate-slug regression**
|
||||
|
||||
Create `default` and `team-x` copies of the origin and target, add `(team-x, origin) -> (team-x, target)` plus the wrong reverse `(team-x, target) -> (default, origin)`, validate with `sourceId: 'team-x'`, and require one warning.
|
||||
|
||||
- [ ] **Step 2: Run the focused test and verify RED**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts -t "wrong-source reverse" > "$TEMP/backlink-red.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: assertion failure because current slug-only validation returns zero findings.
|
||||
|
||||
- [ ] **Step 3: Add the remaining behavioral regressions after the first red is recorded**
|
||||
|
||||
Add tests proving that the exact reverse clears the warning, a legitimate cross-source forward/reverse pair passes, and two destinations sharing one slug but differing by source are validated independently.
|
||||
|
||||
### Task 2: Expose exact endpoint identity from both engines
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/core/types.ts:1204-1229`
|
||||
- Modify: `src/core/postgres-engine.ts:3021-3124`
|
||||
- Modify: `src/core/pglite-engine.ts:2941-3037`
|
||||
- Modify: `test/get-page-federated-scope.test.ts:187-246,289-306`
|
||||
- Modify: `test/e2e/multi-source-bug-class.test.ts:184-205`
|
||||
- Modify: `test/e2e/engine-parity.test.ts:813-875`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `Link.from_source_id: string`, `Link.to_source_id: string`, and `Link.origin_source_id?: string | null`.
|
||||
- Preserves: `getLinks(slug, { sourceId?, sourceIds? })` and `getBacklinks(...)` filtering semantics.
|
||||
|
||||
- [ ] **Step 1: Add engine-contract assertions before implementation**
|
||||
|
||||
Assert scalar cross-source rows expose `beta -> default`, federated rows expose only in-grant endpoint IDs, `sourceIds` still beats scalar `sourceId`, and an out-of-grant origin has both `origin_slug` and `origin_source_id` null.
|
||||
|
||||
- [ ] **Step 2: Run the focused contract tests and verify RED**
|
||||
|
||||
```bash
|
||||
bun test test/get-page-federated-scope.test.ts test/e2e/multi-source-bug-class.test.ts > "$TEMP/link-identity-red.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: source-ID assertions fail because fields are absent.
|
||||
|
||||
- [ ] **Step 3: Extend `Link` and project IDs without changing predicates**
|
||||
|
||||
Use this additive contract:
|
||||
|
||||
```ts
|
||||
export interface Link {
|
||||
from_slug: string;
|
||||
from_source_id: string;
|
||||
to_slug: string;
|
||||
to_source_id: string;
|
||||
link_type: string;
|
||||
context: string;
|
||||
link_source?: string | null;
|
||||
origin_slug?: string | null;
|
||||
origin_source_id?: string | null;
|
||||
origin_field?: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
In all six branches per engine, project:
|
||||
|
||||
```sql
|
||||
f.source_id AS from_source_id,
|
||||
t.source_id AS to_source_id,
|
||||
o.source_id AS origin_source_id
|
||||
```
|
||||
|
||||
Keep every `WHERE` and grant-aware origin `LEFT JOIN` unchanged.
|
||||
|
||||
- [ ] **Step 4: Re-run contract tests and verify GREEN**
|
||||
|
||||
Use the same command and require all focused tests to pass.
|
||||
|
||||
### Task 3: Validate exact reverse identities and propagate scope
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/core/output/writer.ts:89-96,240-318`
|
||||
- Modify: `src/core/output/post-write.ts:36-41,73-118`
|
||||
- Modify: `src/core/output/validators/back-link.ts:24-47`
|
||||
- Modify: `src/core/operations.ts:1227-1246`
|
||||
- Modify: `test/post-write-lint.test.ts:67-130`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: optional `PageValidationContext.sourceId` and `sourceIds`, with `sourceIds` taking precedence.
|
||||
- `runPostWriteLint(..., opts)` accepts the same optional scope and loads the validated page through it.
|
||||
|
||||
- [ ] **Step 1: Add a post-write nested-read regression and verify RED**
|
||||
|
||||
Validate a non-default page with a wrong-source reverse via `runPostWriteLint(..., { force: true, noLog: true, sourceId: 'team-x' })`; require a backlink warning.
|
||||
|
||||
- [ ] **Step 2: Implement minimal scope propagation**
|
||||
|
||||
Add `sourceId?`/`sourceIds?` to validation context and lint options. Load pages using `sourceIds` when non-empty, otherwise scalar `sourceId`. Pass the same scope into nested validators. In the put-page success hook, call lint with the already-resolved write source ID.
|
||||
|
||||
- [ ] **Step 3: Implement exact backlink matching**
|
||||
|
||||
Initial outbound reads use the validation scope. Deduplicate rows by all four endpoint identity fields so every distinct expected origin remains represented even when targets share a source-qualified identity. Read each target using the federated grant when present, otherwise the target's exact scalar source. Accept only a row matching all four endpoint fields of the expected reverse.
|
||||
|
||||
- [ ] **Step 4: Run writer and post-write tests and verify GREEN**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts test/post-write-lint.test.ts > "$TEMP/backlink-green.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: all tests pass, including the recorded false-negative.
|
||||
|
||||
### Task 4: Verify PostgreSQL/PGLite parity and final scope
|
||||
|
||||
**Files:**
|
||||
- Modify: `test/e2e/engine-parity.test.ts:813-875`
|
||||
- Verify: all files above
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: exact endpoint fields and unchanged filtering semantics.
|
||||
- Produces: parity evidence for scalar cross-source and federated reads.
|
||||
|
||||
- [ ] **Step 1: Compare complete endpoint tuples across engines**
|
||||
|
||||
Compare sorted tuples containing `from_source_id`, `from_slug`, `to_source_id`, `to_slug`, `origin_source_id`, and `origin_slug` for scalar and federated fixtures.
|
||||
|
||||
- [ ] **Step 2: Run focused PGLite/source-isolation tests**
|
||||
|
||||
```bash
|
||||
bun test test/writer.test.ts test/post-write-lint.test.ts test/get-page-federated-scope.test.ts test/e2e/multi-source-bug-class.test.ts > "$TEMP/backlink-focused.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0.
|
||||
|
||||
- [ ] **Step 3: Run PostgreSQL parity when the test database is available**
|
||||
|
||||
```bash
|
||||
bun test test/e2e/engine-parity.test.ts -t "federated sourceIds" --timeout=300000 > "$TEMP/backlink-parity.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0; if the configured test database is unavailable, report the exact environmental blocker rather than claiming parity execution.
|
||||
|
||||
- [ ] **Step 4: Typecheck and inspect the final diff**
|
||||
|
||||
```bash
|
||||
bun run typecheck > "$TEMP/backlink-typecheck.txt" 2>&1
|
||||
```
|
||||
|
||||
Expected: exit 0. Then run `git diff --check` and confirm no version, schema, migration, deployment, or conditional-write files changed.
|
||||
@@ -0,0 +1,142 @@
|
||||
# Engine dynamic-import reconciliation design
|
||||
|
||||
**Date:** 2026-07-28
|
||||
|
||||
## Goal
|
||||
|
||||
Reconcile the overlapping engine dynamic-import changes from:
|
||||
|
||||
- `claude/hungry-edison-8bb1cd` at release commits `48ada48f` and `248bfe55`
|
||||
- `claude/elegant-gates-e5275e` at `ef4cf7a8`
|
||||
|
||||
onto a fresh branch from current `origin/master`, without merging or cherry-picking either lineage wholesale and without adding a release/version bump.
|
||||
|
||||
## Established state
|
||||
|
||||
At investigation time:
|
||||
|
||||
- `origin/master` was `6136e139972a5449630b4f47f5ed7b4cbe5b811b`, version `0.42.67.0`.
|
||||
- Upstream PR #3511 was still open, so trunk did not contain its two `chronicle/ontology.ts` hoists.
|
||||
- Neither source branch was an ancestor of trunk.
|
||||
- Trunk contained 17 dynamic imports in the three engine-path files:
|
||||
- 13 safe-hoist candidates: two ontology imports, nine engine helper/audit imports, and two migration imports.
|
||||
- Four `ai/gateway.ts` imports, all inside `try/catch` fallback paths.
|
||||
- `git log -G` showed the separate ontology, helper, migration, and gateway histories. `git log -S` is not suitable for this dynamic-to-static replacement because the relevant token can remain present while its context changes.
|
||||
- The guard from `ef4cf7a8` passed against that commit but failed against trunk. It also knew about only two gateway opt-outs because two `_upsertChunksOnce` gateway lookups landed later in trunk.
|
||||
|
||||
## Selected approach
|
||||
|
||||
Reconstruct the intended current state directly on fresh `origin/master`.
|
||||
|
||||
Do not merge or cherry-pick either old lineage. Selectively reproduce the desired source changes, adapt the guard to the current four gateway call sites, and write current-state documentation. This avoids importing stale release metadata, stale TODO claims, and unrelated lineage changes.
|
||||
|
||||
## Source changes
|
||||
|
||||
### Safe static imports
|
||||
|
||||
Hoist all 13 safe candidates:
|
||||
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `valueHash`, `normalizeDimension`, `isNovelDimension` from `chronicle/ontology.ts`
|
||||
- `isRetryableConnError` through the existing `retry.ts` import
|
||||
- `resolveRecencyDecayMap`, `DEFAULT_FALLBACK` from `search/recency-decay.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- the same ontology, retry, and recency helpers
|
||||
- `isConnectionEndedError` from `retry-matcher.ts`
|
||||
- `logDbDisconnect` from `audit/db-disconnect-audit.ts`
|
||||
- `logPoolRecovery` from `audit/pool-recovery-audit.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- `isStatementTimeoutError`, `isRetryableConnError` from `retry-matcher.ts`
|
||||
- `repairTimelineDedupIndex` from `timeline-dedup-repair.ts`
|
||||
|
||||
The implementation must keep the two engines in parity where the behavior is shared. Comments should describe current invariants, not repeat an unproven causal claim that these hoists fix the Windows test-runner crash.
|
||||
|
||||
### Deliberately lazy gateway imports
|
||||
|
||||
Keep all four `await import('./ai/gateway.ts')` call sites lazy:
|
||||
|
||||
- PGLite `initSchema`
|
||||
- PGLite `_upsertChunksOnce`
|
||||
- Postgres `initSchema`
|
||||
- Postgres `_upsertChunksOnce`
|
||||
|
||||
Each line receives the explicit `engine-dynamic-import-ok` marker and a concise nearby rationale.
|
||||
|
||||
The rationale has two parts:
|
||||
|
||||
1. The gateway's static closure includes the AI SDK, provider packages, and validation/config machinery, so eager loading would tax engine startup paths that do not otherwise need it.
|
||||
2. More importantly, each lookup is inside a `try/catch` that preserves a soft fallback (compiled defaults or the brain's stored embedding-model config). Hoisting the module would evaluate it before that catch can run and could convert a recoverable configuration/import failure into a module-load-time hard failure.
|
||||
|
||||
The guard must not allow unmarked gateway imports or a broad file-level exemption.
|
||||
|
||||
## Guard and wiring
|
||||
|
||||
Add `scripts/check-engine-dynamic-import.sh`, adapted from `ef4cf7a8`, with these properties:
|
||||
|
||||
- Default scan set:
|
||||
- `src/core/pglite-engine.ts`
|
||||
- `src/core/postgres-engine.ts`
|
||||
- `src/core/migrate.ts`
|
||||
- Normalize trailing CR before matching so CRLF checkouts cannot bypass the check.
|
||||
- Ignore comment-only lines.
|
||||
- Ignore only lines carrying `engine-dynamic-import-ok`.
|
||||
- Report every unmarked `await import(` with file and line.
|
||||
- Explain that contributors should prefer a static import and must justify a real opt-out.
|
||||
- Avoid asserting that every dynamic import deterministically crashes Windows; the measured evidence supports treating the pattern as an engine-path hardening invariant, while box-level commit exhaustion remained a confound in prior runs.
|
||||
|
||||
Wire it into:
|
||||
|
||||
- `package.json` as `check:engine-dynamic-import`
|
||||
- `package.json` `check:all`
|
||||
- `scripts/run-verify-parallel.sh`
|
||||
|
||||
Follow trunk's current rule that package scripts invoke repository shell scripts through `bash`.
|
||||
|
||||
## Regression coverage
|
||||
|
||||
Add an automated test for the guard. It must cover:
|
||||
|
||||
- A real dynamic import produces exit 1 and is reported.
|
||||
- A line carrying `engine-dynamic-import-ok` is allowed.
|
||||
- Line comments and block-comment lines do not produce findings.
|
||||
- The same violation is caught with CRLF input.
|
||||
- The default repository scan passes after the source reconciliation.
|
||||
|
||||
Use a temporary fixture rather than mutating tracked source files. Keep assertions path-portable.
|
||||
|
||||
The pre-fix red demonstration is the exact guard from `ef4cf7a8` run against current trunk: it exits 1 and reports the existing unmarked imports. The post-fix guard and test must pass.
|
||||
|
||||
## Documentation policy
|
||||
|
||||
Preserve current behavior, not either old release narrative:
|
||||
|
||||
- Do not modify `VERSION` or add a release `CHANGELOG.md` entry.
|
||||
- Do not copy old version headings or completed release TODO blocks.
|
||||
- Do not retain the old TODO claiming that extracting gateway accessors is necessarily the fix; the lazy imports are deliberately protected by their local soft-failure boundaries.
|
||||
- Add the cross-cutting no-unmarked-dynamic-import invariant to `CLAUDE.md`.
|
||||
- Update the current-state entries for `src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and `src/core/migrate.ts` in `docs/architecture/KEY_FILES.md` where needed.
|
||||
- Regenerate `llms.txt` and `llms-full.txt` after the documentation edits.
|
||||
- Add a TODO only if implementation uncovers a real unresolved action.
|
||||
|
||||
Public documentation must use generic language and must not overstate the historical Windows crash causality.
|
||||
|
||||
## Verification
|
||||
|
||||
Capture full output to files before inspecting summaries. Run, at minimum:
|
||||
|
||||
1. The guard regression test.
|
||||
2. `bash scripts/check-engine-dynamic-import.sh`.
|
||||
3. Focused tests that exercise the touched engine, migration, retry, audit, and recency modules.
|
||||
4. `bun run typecheck`.
|
||||
5. `bun run verify`.
|
||||
6. `bun run build:llms` followed by `bun test test/build-llms.test.ts`.
|
||||
7. `git diff --check` and a final clean-status/diff review.
|
||||
|
||||
If platform contention or existing Windows suite defects block a broad test, report the exact command, exit code, and ownership classification rather than declaring success from a partial run.
|
||||
|
||||
## Git and publication boundary
|
||||
|
||||
- Work on `claude/kind-meitner-330c90`, reset locally to the exact investigated `origin/master` base.
|
||||
- Preserve the previous worktree tip under `claude/kind-meitner-330c90-pre-reconcile`.
|
||||
- Keep implementation and verification commits local.
|
||||
- Do not push, create a PR, comment upstream, or otherwise publish without explicit user approval after the local result is complete.
|
||||
@@ -0,0 +1,184 @@
|
||||
# Scalar-source backlink validation design
|
||||
|
||||
## Problem
|
||||
|
||||
A page identity in a multi-source brain is `(source_id, slug)`, but the back-link validator currently reasons only about `slug`.
|
||||
|
||||
For an outbound edge:
|
||||
|
||||
```text
|
||||
(source-a, concepts/origin) -> (source-a, people/target)
|
||||
```
|
||||
|
||||
the validator accepts any reverse row whose bare slugs are:
|
||||
|
||||
```text
|
||||
people/target -> concepts/origin
|
||||
```
|
||||
|
||||
That can incorrectly accept a row ending at `(default, concepts/origin)` instead of `(source-a, concepts/origin)`.
|
||||
|
||||
The bug is not that scalar `getLinks(slug, { sourceId })` permits cross-source destinations. That behavior is intentional: scalar scope qualifies the near/from endpoint while trusted local callers retain visibility into explicit cross-source edges. The gap is that a returned `Link` does not carry the source identity of either endpoint, so callers cannot distinguish same-slug pages.
|
||||
|
||||
## Reproduction and evidence
|
||||
|
||||
A deterministic PGLite reproduction creates duplicate `concepts/a` and `people/b` pages in `default` and `team-x`, then adds:
|
||||
|
||||
```text
|
||||
(team-x, concepts/a) -> (team-x, people/b)
|
||||
(team-x, people/b) -> (default, concepts/a)
|
||||
```
|
||||
|
||||
The second edge is not a valid reverse of the first. Nevertheless:
|
||||
|
||||
```ts
|
||||
await engine.getLinks('people/b', { sourceId: 'team-x' })
|
||||
```
|
||||
|
||||
returns the second row, and the current validator accepts it because `to_slug === 'concepts/a'`.
|
||||
|
||||
Both engines implement the same scalar rule: filter `f.slug` and `f.source_id`, join the actual destination by `to_page_id`, and do not filter `t.source_id`. Federated `sourceIds` is a separate branch that constrains all visible endpoints and takes precedence over scalar scope.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Validate back-links by exact source-qualified endpoint identity.
|
||||
2. Preserve explicit cross-source links for trusted scalar reads.
|
||||
3. Preserve federated all-endpoint containment and `sourceIds` precedence.
|
||||
4. Keep PostgreSQL and PGLite behavior identical.
|
||||
5. Add strict red-before-green regressions using duplicate slugs across sources.
|
||||
6. Avoid schema migrations and production operational changes.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing scalar link reads to same-source-only reads.
|
||||
- Weakening or widening federated reads.
|
||||
- Changing link write identity or database schema.
|
||||
- Refactoring the atomic conditional-write branch.
|
||||
- Coupling deployment, restart, or migration mechanics to the backlink code change. Release operations are handled separately after verification.
|
||||
|
||||
## Chosen approach
|
||||
|
||||
Extend the engine `Link` result with endpoint source identities and use those fields in the validator.
|
||||
|
||||
```ts
|
||||
interface Link {
|
||||
from_slug: string;
|
||||
from_source_id: string;
|
||||
to_slug: string;
|
||||
to_source_id: string;
|
||||
// existing fields
|
||||
origin_slug?: string | null;
|
||||
origin_source_id?: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
All `getLinks` and `getBacklinks` query branches in PostgreSQL and PGLite will project the source IDs from the pages already joined as `f`, `t`, and `o`. No filtering behavior changes.
|
||||
|
||||
This approach is preferred over a dedicated `hasExactLink` method because it keeps source identity attached to the link data everywhere, avoids duplicate engine SQL and per-edge existence queries, and matches existing source-qualified link-write and batch-row contracts.
|
||||
|
||||
Validator-only raw SQL is rejected because validators should consume the `BrainEngine` contract rather than bypass it with engine-specific schema knowledge.
|
||||
|
||||
## Engine semantics
|
||||
|
||||
The existing three read modes remain unchanged.
|
||||
|
||||
### Unscoped
|
||||
|
||||
`getLinks(slug)` returns rows from all same-slug from-pages across sources. Each row identifies the actual source of both endpoints.
|
||||
|
||||
### Scalar source
|
||||
|
||||
`getLinks(slug, { sourceId })` matches exactly `(sourceId, slug)` on the from side. A destination may belong to another source, and `to_source_id` reveals that exact identity.
|
||||
|
||||
The corresponding scalar `getBacklinks` rule continues to match the exact destination/to-page identity while allowing a cross-source referrer.
|
||||
|
||||
### Federated sources
|
||||
|
||||
`getLinks(slug, { sourceIds })` continues to constrain from and to endpoints to the grant. The origin join continues to redact an out-of-grant origin. `sourceIds` continues to take precedence over scalar `sourceId`.
|
||||
|
||||
Adding source IDs to returned in-grant endpoints does not disclose anything new: the existing result already discloses those pages' slugs and edges. An out-of-grant endpoint remains absent.
|
||||
|
||||
## Validator algorithm
|
||||
|
||||
The validator receives the source scope associated with the page being validated.
|
||||
|
||||
For every outbound edge:
|
||||
|
||||
```text
|
||||
(from_source_id, from_slug) -> (to_source_id, to_slug)
|
||||
```
|
||||
|
||||
it requires a reverse row:
|
||||
|
||||
```text
|
||||
(to_source_id, to_slug) -> (from_source_id, from_slug)
|
||||
```
|
||||
|
||||
Duplicate edge rows are deduplicated by the full endpoint pair `(from_source_id, from_slug, to_source_id, to_slug)`, not by bare target slug. This preserves separate reverse requirements when multiple same-slug origin pages point to one exact target.
|
||||
|
||||
For each target:
|
||||
|
||||
1. Read target outbound links using the target's exact scalar source when validation is scalar-scoped.
|
||||
2. Under federated validation, retain the caller's `sourceIds` grant rather than converting it to scalar scope.
|
||||
3. Accept only a returned row whose `from_source_id`, `from_slug`, `to_source_id`, and `to_slug` exactly match the expected reverse identity.
|
||||
4. Emit the existing warning when no exact reverse exists.
|
||||
|
||||
This preserves legitimate cross-source pairs. For example:
|
||||
|
||||
```text
|
||||
(source-a, concepts/origin) -> (source-b, people/target)
|
||||
(source-b, people/target) -> (source-a, concepts/origin)
|
||||
```
|
||||
|
||||
is valid.
|
||||
|
||||
## Validation context propagation
|
||||
|
||||
`PageValidationContext` must carry the relevant scalar or federated source scope. The writer and post-write lint paths must load the page with that scope and pass the same scope to nested validator reads.
|
||||
|
||||
This change is scoped to source routing needed by validation. It does not modify conditional-write revision or conflict semantics and must not be applied to the atomic conditional-write branch.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
### PGLite strict-TDD regression
|
||||
|
||||
Add duplicate pages across `default` and a second source, then prove before the production fix that:
|
||||
|
||||
1. A forward edge in the second source plus a wrong-source reverse produces a warning.
|
||||
2. Adding the exact reverse removes the warning.
|
||||
3. A legitimate cross-source forward/reverse pair passes.
|
||||
4. Two same-slug destination pages are not collapsed into one target identity.
|
||||
|
||||
The first assertion must fail against the pre-fix implementation.
|
||||
|
||||
### Engine contract tests
|
||||
|
||||
For PGLite and PostgreSQL:
|
||||
|
||||
1. Assert link rows expose exact from/to source IDs.
|
||||
2. Assert scalar reads still return explicit cross-source destinations.
|
||||
3. Assert federated reads still exclude out-of-grant endpoints.
|
||||
4. Assert `sourceIds` still takes precedence over scalar `sourceId`.
|
||||
5. Assert origin source identity is null when the origin is redacted by the federated branch.
|
||||
|
||||
### Parity and focused verification
|
||||
|
||||
Run:
|
||||
|
||||
- the focused backlink validator test;
|
||||
- source-isolation and federated link tests;
|
||||
- the Postgres/PGLite parity fixture with a test database;
|
||||
- related writer/post-write tests;
|
||||
- `bun run typecheck`.
|
||||
|
||||
Capture complete command output to files before inspecting summaries. Do not use production databases or restart the live service.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The `Link` change is additive at runtime. Existing consumers that read only slug or provenance fields continue to work. TypeScript object literals typed as complete `Link` values may need source fields; if compatibility pressure is high, the source fields can initially be optional in the public type while engine implementations and validator tests require their presence. The preferred contract is required endpoint source IDs because every persisted link always has both pages and therefore both source IDs.
|
||||
|
||||
No schema migration is required because source IDs already live on the joined `pages` rows.
|
||||
|
||||
## Operational constraints
|
||||
|
||||
The implementation phase does not deploy, restart GBrain, run production migrations, or alter the atomic conditional-write branch. Release, migration, and restart operations are a separate verified workflow and do not change this design's engine or validator semantics.
|
||||
@@ -91,3 +91,18 @@ First full takes extraction run on a ~100K-page brain:
|
||||
4. **Self-reported ≠ verified.** "Reports 7 figures" → holder=person, weight=0.75, NOT world/1.0
|
||||
5. **No false precision.** Use 0.05 increments (0.35, 0.55, 0.75), not 0.74 or 0.82
|
||||
6. **"So what" test.** Skip Twitter handles, follower counts, obvious metadata
|
||||
|
||||
## Owner-holder canonicalization
|
||||
|
||||
"The brain owner" is, by convention, the holder string **`self`** — the value the
|
||||
dream `consolidate` phase stamps when it promotes the owner's hot facts into cold
|
||||
takes. Calibration, `think`, and the `doctor` calibration check resolve the owner
|
||||
holder through `resolveOwnerHolder` (`src/core/owner-holder.ts`): explicit override
|
||||
> `emotional_weight.user_holder` config > `self`.
|
||||
|
||||
Known limitation (tracked in garrytan/gbrain#2465): the owner can also
|
||||
appear under `brain` (a take the owner asserts, via `propose_takes`) and
|
||||
`people/<owner>` (extraction that names the owner). The resolver selects the
|
||||
*default* canonical owner string for reads; it does not merge those other
|
||||
strings. Per-take attribution for other people (e.g. `people/george`) is
|
||||
unaffected and correct.
|
||||
|
||||
@@ -13,7 +13,7 @@ Step-by-step walkthroughs that take you from zero to a working outcome. Concrete
|
||||
|
||||
These are the next tutorials on the roadmap. Open an issue if one of them is the one you need most; that's how we'll prioritize.
|
||||
|
||||
- **Set up GBrain for VC dealflow** — the operator's recipe. People pages for founders, companies with typed Facts fence carrying ARR / team-size / runway across dates, meetings auto-ingested, deal pages linking everything. Shows `gbrain whoknows`, `gbrain find_trajectory`, and `gbrain founder scorecard` on real workflows.
|
||||
- **Set up GBrain for VC dealflow** — the operator's recipe. People pages for founders, companies with typed Facts fence carrying ARR / team-size / runway across dates, meetings auto-ingested, deal pages linking everything. Shows `gbrain whoknows`, `gbrain find-trajectory`, and `gbrain founder scorecard` on real workflows.
|
||||
|
||||
- **Migrate your existing vault into GBrain** — for Notion / Obsidian / Roam users with a vault that doesn't match GBrain's default layout. Walks through `gbrain schema detect` → `suggest` → `review-candidates` so the brain learns your shape instead of forcing you to learn its.
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ There are two ways to scope teammates' access. They suit different deployment sh
|
||||
|
||||
**Model A: separate sources with OAuth scoping (recommended for true multi-user with different AI clients).** What this tutorial walks you through. Each teammate gets their own OAuth client, which carries `--source` + `--federated-read` flags. The brain refuses cross-source reads at the SQL layer; isolation is database-enforced. Each teammate can run their own MCP-aware client (Claude Code, Cursor, their own OpenClaw, etc.) and the scoping holds.
|
||||
|
||||
**Model B: one source, directory-based per-person scoping (simpler for one-agent-serves-everyone setups).** The shape I actually run in production: a single source called `default`, with a `partners/<slug>/` convention inside it (e.g. `partners/alice-example/`, `partners/bob-example/`). Each partner gets their own subdirectory holding their personal pages: `partners/alice-example/USER.md`, `partners/alice-example/concepts/`, `partners/alice-example/sources/`, etc. There's no OAuth-enforced isolation; the agent itself enforces "Alice's writes go to her partners/ subdir." This is the right model when ONE agent (yours) serves everyone over Telegram or a single shared interface. It's simpler ops, no per-user OAuth, but the scoping is convention-only.
|
||||
**Model B: one source, directory-based per-person scoping (simpler for one-agent-serves-everyone setups).** The shape I actually run in production: a single source called `default`, with a `partners/<slug>/` convention inside it (e.g. `partners/alice-example/`, `partners/bob-example/`). Each partner gets their own subdirectory holding their personal pages: `partners/alice-example/USER.md`, `partners/alice-example/concepts/`, `partners/alice-example/sources/`, etc. This is the right model when ONE agent (yours) serves everyone over Telegram or a single shared interface. It's simpler ops, no per-user OAuth. **Write scoping within the shared source can be server-enforced:** register each per-person client with `--bound-slug-prefixes partners/alice-example/` and every slug-mutating write outside that prefix is rejected with `permission_denied` (v0.42.72.0+). Without the binding, the scoping is convention-only (the agent polices itself). Read scoping stays source-granular in both models — within a shared source, everyone entitled to the source can read every folder.
|
||||
|
||||
For most company-brain installs (10+ teammates each with their own AI client), Model A is the right starting point. If you're running the fat-agent-serves-everyone pattern from the personal-brain tutorial, Model B is genuinely simpler. You can also mix: separate sources for the obviously-different ones (customer notes vs internal-only) AND a `partners/<slug>/` convention inside the shared source for per-person workspace.
|
||||
|
||||
@@ -158,7 +158,7 @@ gbrain serve --http --port 3131 --bind 0.0.0.0
|
||||
|
||||
The `--bind 0.0.0.0` is important. By default the server binds to localhost only, which is correct for a personal install but blocks remote teammates. Setting `0.0.0.0` accepts connections from any interface.
|
||||
|
||||
The server prints an admin bootstrap token to stderr on first start. Save it. You'll use it once for the admin dashboard.
|
||||
The server prints an admin bootstrap token to stderr on first start when run in an interactive terminal. Save it. You'll use it once for the admin dashboard. On a non-TTY start (systemd, Docker, piped logs) the token is hidden from logs — set `GBRAIN_ADMIN_BOOTSTRAP_TOKEN` yourself or pass `--print-admin-token` on a trusted terminal instead.
|
||||
|
||||
For development, tunnel the local server out via ngrok:
|
||||
|
||||
@@ -210,7 +210,7 @@ Each `register-client` command prints a `client_id` and a `client_secret`. Save
|
||||
A note on the flags:
|
||||
|
||||
- `--scopes read,write` lets the client query the brain and write new pages. You can omit `write` for read-only clients (executive summaries, dashboards). The `admin` scope is needed for operational commands like `gbrain remote doctor` and is usually reserved for your own admin client.
|
||||
- `--source` controls write authority. A client can only write to one source. Within that source, your folder convention from Part 3 keeps each person's writes in their own subfolder.
|
||||
- `--source` controls write authority. A client can only write to one source. Within that source, your folder convention from Part 3 keeps each person's writes in their own subfolder — and you can make that server-enforced with `--bound-slug-prefixes alice-example/` (v0.42.72.0+): every slug-mutating write op (put_page, delete_page, tags, links, timeline, revert, raw data) outside the bound prefixes is rejected with `permission_denied`. Update the binding later with `gbrain auth rescope-client <id> --bound-slug-prefixes <p1,p2|none>`. **Adding a binding to an existing client narrows it in ways you should expect:** ops that write by something other than a slug (`extract_entities`, `extract_facts`, `forget_fact`, `ontology_propose`, `sources_add`/`sources_remove`) and `POST /ingest` become unavailable to that client, and `put_page`'s automatic fact extraction is skipped — all because none of them can be confined to a prefix. Reads are unaffected. See [the qm-harness guide](../integrations/qm-harness.md) for the full model.
|
||||
- `--federated-read` controls read scope. A client can read from one or more sources.
|
||||
|
||||
### Verify the scoping actually scopes
|
||||
@@ -484,6 +484,10 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho
|
||||
|
||||
The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard.
|
||||
|
||||
### If agents run as containers on the same Docker host
|
||||
|
||||
OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
---
|
||||
|
||||
## Part 13: Cost and speed expectations
|
||||
@@ -550,7 +554,7 @@ What to do next:
|
||||
|
||||
- **Wire ingestion** from external systems (Granola, Linear, Slack) using the [ingestion source contract](../skillpack-anatomy.md). Most companies want their meetings auto-ingested so the brain stays current without anyone typing notes.
|
||||
- **Set up team-specific dashboards** through the admin UI. Each team lead can have their own view of brain health and activity.
|
||||
- **Explore the rest of the brain layer.** `gbrain whoknows` (find the expert on a topic), `gbrain find_trajectory` (how a metric changed over time), `gbrain founder scorecard` (especially useful for VC and ops teams), the contradiction-detection cycle that surfaces conflicts between different people's notes.
|
||||
- **Explore the rest of the brain layer.** `gbrain whoknows` (find the expert on a topic), `gbrain find-trajectory` (how a metric changed over time), `gbrain founder scorecard` (especially useful for VC and ops teams), the contradiction-detection cycle that surfaces conflicts between different people's notes.
|
||||
|
||||
If you're building in this space (which YC has flagged as the [company-brain category in its Request for Startups](https://www.ycombinator.com/rfs#company-brain)), you might as well build on this. Everything described above is open source, MIT licensed, and what I run in production behind my own AI agents.
|
||||
|
||||
|
||||
@@ -233,13 +233,14 @@ keep it or `git checkout` to throw it away. Nothing is committed for you.
|
||||
|
||||
**For a skill that ships with gbrain** (anything under the gbrain repo's own
|
||||
`skills/`): SkillOpt refuses to overwrite it by default and writes the winner to
|
||||
`skills/<name>/skillopt/best.md` instead, so an optimization pass can never
|
||||
silently mutate a skill other people depend on. Two ways to handle that:
|
||||
`skills/<name>/skillopt/proposed.md` instead (while keeping `best.md` as the
|
||||
optimizer's current-best pointer), so an optimization pass can never silently
|
||||
mutate a skill other people depend on. Two ways to handle that:
|
||||
|
||||
```bash
|
||||
# See the proposed improvement without touching SKILL.md (works for ANY skill):
|
||||
gbrain skillopt meeting-prep --split 1:1:1 --no-mutate
|
||||
# → writes skills/meeting-prep/skillopt/best.md (the proposed rewrite), prints its path. Copy what you want.
|
||||
# → writes skills/meeting-prep/skillopt/proposed.md, updates best.md, and prints the proposal path.
|
||||
|
||||
# Actually rewrite a bundled skill (explicit opt-in + an independent held-out set):
|
||||
gbrain skillopt brain-ops --split 1:1:1 --allow-mutate-bundled \
|
||||
|
||||
@@ -115,21 +115,21 @@ You can use the same keys across multiple agents.
|
||||
|
||||
## Step 6: Install GBrain
|
||||
|
||||
Once OpenClaw is running:
|
||||
Once OpenClaw is running, installation is two commands — one in the brain repo, one in the agent workspace:
|
||||
|
||||
```bash
|
||||
gbrain install
|
||||
# In the BRAIN repo (the git repo that holds your markdown pages):
|
||||
gbrain init --supabase
|
||||
|
||||
# In the AGENT WORKSPACE repo (where OpenClaw runs):
|
||||
gbrain skillpack scaffold --all
|
||||
```
|
||||
|
||||
This installs:
|
||||
`gbrain init --supabase` walks a short wizard that asks for your Supabase connection string and creates the schema. You'll get that connection string in Step 7 — read 7a and 7b first so you paste the right one (the transaction pooler, not the direct connection). If you'd rather try things locally before paying for a database, `gbrain init --pglite` gives you a zero-config embedded engine instead; you can migrate to Supabase later with `gbrain migrate --to supabase`.
|
||||
|
||||
- About 60 skills
|
||||
- About 9 skill packs
|
||||
- Default brain structure
|
||||
- MCP server configuration
|
||||
- Supabase connection (for embeddings and search)
|
||||
`gbrain skillpack scaffold --all` copies the ~43 bundled skills into your agent workspace as first-class files you can edit freely. (The old managed-install model was retired in v0.36.0.0; see `docs/INSTALL.md` if you're upgrading from an older release.)
|
||||
|
||||
GBrain populates the brain repo with its default directory structure, skill files, and configuration. From this point, the agent has working memory and access to every skill.
|
||||
From this point, the agent has working memory and access to every skill.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ gbrain schema sync --apply
|
||||
The sync backfills `page.type = 'meeting'` on all 4000 pages in 1000-row batches. Now:
|
||||
|
||||
- `gbrain whoknows "Q3 roadmap discussion"` routes through the meeting type, ranking by `expert_routing` signal (attendees, recency, salience) instead of raw text.
|
||||
- `gbrain extract-facts` runs on every meeting page automatically (because `extractable: true`), pulling typed facts like `attended_by=alice-example`, `date=2026-05-23`.
|
||||
- The `extract_facts` cycle runs on every meeting page automatically (because `extractable: true`), pulling typed facts like `attended_by=alice-example`, `date=2026-05-23`.
|
||||
- The downstream `think` skill can now answer "what did we decide about pricing in the last three roadmap meetings" by querying the meeting graph instead of grep'ing 4000 files.
|
||||
|
||||
One command. 4000 pages went from invisible to queryable. The content didn't change. The structure did.
|
||||
@@ -62,7 +62,7 @@ gbrain schema add-link-type led-by --page-type deal --target-type inves
|
||||
gbrain schema sync --apply
|
||||
```
|
||||
|
||||
Now `gbrain whoknows "Series A SaaS"` routes through `investor` and `portco` types specifically, not the noisy general type set. `gbrain graph-query alice-example --type intro-from --depth 2` walks two hops of intros to surface "Alice introduced you to Bob who introduced you to Charlie." `gbrain extract-facts` starts producing typed claims from the fence in your deal pages: `(deals/acme-seed, raise=2000000, valuation=15000000, lead=widget-vc, closed_at=2026-05-23)`.
|
||||
Now `gbrain whoknows "Series A SaaS"` routes through `investor` and `portco` types specifically, not the noisy general type set. `gbrain graph-query alice-example --type intro-from --depth 2` walks two hops of intros to surface "Alice introduced you to Bob who introduced you to Charlie." The `extract_facts` cycle starts producing typed claims from the fence in your deal pages: `(deals/acme-seed, raise=2000000, valuation=15000000, lead=widget-vc, closed_at=2026-05-23)`.
|
||||
|
||||
The CRM you've been promising yourself you'll set up next quarter? You just shipped it in 4 commands. It's downstream of your notes, not parallel to them.
|
||||
|
||||
@@ -143,7 +143,7 @@ Re-run the same `whoknows` query. Top-3 should shift, because the new type is no
|
||||
|
||||
Three things gbrain does that generic note systems can't:
|
||||
|
||||
**1. The brain knows the difference between a person and an idea.** Page-type matters at query time. `gbrain whoknows` only considers `expert_routing: true` types. `gbrain extract-facts` only runs on `extractable: true` types. `gbrain graph-query` walks declared link verbs. None of that works on a flat tag system because tags don't have semantics — they're labels. Types are first-class citizens with rules attached.
|
||||
**1. The brain knows the difference between a person and an idea.** Page-type matters at query time. `gbrain whoknows` only considers `expert_routing: true` types. The `extract_facts` cycle only runs on `extractable: true` types. `gbrain graph-query` walks declared link verbs. None of that works on a flat tag system because tags don't have semantics — they're labels. Types are first-class citizens with rules attached.
|
||||
|
||||
**2. Untyped content is invisible content.** If your meetings are typed as `note`, expert routing skips them, facts extraction ignores them, link inference doesn't fire. They exist on disk and they're indexed for text search, but the structural surfaces (whoknows, find_experts, recall, think) treat them as second-class. Adding a type isn't cosmetic; it's structural promotion.
|
||||
|
||||
|
||||
@@ -415,6 +415,7 @@ export async function main(argv: string[]): Promise<number> {
|
||||
chat_model: config?.chat_model ?? modelFull,
|
||||
chat_fallback_chain: config?.chat_fallback_chain,
|
||||
base_urls: config?.provider_base_urls,
|
||||
provider_chat_options: config?.provider_chat_options,
|
||||
env: { ...process.env } as Record<string, string>,
|
||||
});
|
||||
|
||||
|
||||
+165
-13
@@ -216,6 +216,19 @@ Per-file detail is in `docs/architecture/KEY_FILES.md`.
|
||||
text, the cast parses it). Guarded by `scripts/check-jsonb-pattern.sh` (template grep) +
|
||||
`scripts/check-jsonb-params.mjs` (positional AST scanner); the real backstop is the DATABASE_URL-gated
|
||||
e2e parity tests, since PGLite can't surface the bug. Full rule in `docs/ENGINES.md`.
|
||||
- **Engine-live paths avoid runtime dynamic `import()` for helper dependencies.** In
|
||||
`src/core/pglite-engine.ts`, `src/core/postgres-engine.ts`, and
|
||||
`src/core/migrate.ts`, dependencies previously reached through runtime dynamic
|
||||
imports use static top-level imports. The only current dynamic-`import()` exceptions
|
||||
are the four `ai/gateway.ts` lookups in both engines'
|
||||
`initSchema()` and `_upsertChunksOnce()` methods; each remains lazy inside a
|
||||
local `try/catch` because the gateway has a large provider/config closure and,
|
||||
more importantly, eager evaluation would occur before the catch and could
|
||||
turn a recoverable default/config-row fallback into a module-load failure.
|
||||
Every exception carries `engine-dynamic-import-ok` on the import line.
|
||||
`scripts/check-engine-dynamic-import.sh` enforces the rule. For history, use
|
||||
`git log -G'await[[:space:]]+import\\('`, not `git log -S`: a dynamic-to-static
|
||||
rewrite can preserve the searched token while changing its context.
|
||||
- **Engine parity.** `src/core/postgres-engine.ts` and `src/core/pglite-engine.ts` move in
|
||||
lockstep — a new method/SQL shape lands in BOTH, pinned by `test/e2e/engine-parity.test.ts`.
|
||||
Forward-referenced columns/indexes go in the bootstrap probe set (guarded by
|
||||
@@ -1006,6 +1019,13 @@ If you fetched this file by URL without cloning yet, the companion files live at
|
||||
|
||||
## Step 1: Install GBrain
|
||||
|
||||
> **NEVER install from the npm registry.** GBrain is not distributed on npm; the npm
|
||||
> package named `gbrain` is an unrelated package. Do NOT run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` (note the missing `github:` prefix — that's the trap). The only
|
||||
> supported sources are `github:garrytan/gbrain` and a git clone, exactly as shown below.
|
||||
> If an unrelated npm install is already present, remove it first
|
||||
> (`npm uninstall -g gbrain` / `bun remove -g gbrain`); `gbrain doctor` also detects this.
|
||||
|
||||
Default path (Bun is required — gbrain is a Bun + TypeScript runtime):
|
||||
|
||||
```bash
|
||||
@@ -1559,14 +1579,24 @@ This is the difference between a search engine and a brain. Search finds the pag
|
||||
|
||||
## Install
|
||||
|
||||
> [!WARNING]
|
||||
> **GBrain is NOT distributed on npm.** The npm package named `gbrain` is an unrelated
|
||||
> package with no connection to this project. Do not run `npm install -g gbrain` or
|
||||
> `bun add -g gbrain` — you'll get something else, and it can shadow the real binary on
|
||||
> your PATH. Install and upgrade ONLY via the documented paths below
|
||||
> (`bun install -g github:garrytan/gbrain`, or `git clone` + `bun install && bun link`).
|
||||
> If you already ran the npm install by mistake: `npm uninstall -g gbrain` /
|
||||
> `bun remove -g gbrain`, then reinstall from GitHub. `gbrain doctor` detects a
|
||||
> shadowing npm install and prints the fix.
|
||||
|
||||
GBrain is designed to be installed and operated by an AI agent. The fastest path is to have your agent do it for you. The CLI and MCP paths below are for people who want to wire it up themselves.
|
||||
|
||||
### Have your agent install it (recommended)
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
@@ -1702,7 +1732,7 @@ Most personal-knowledge tools force one fixed layout: their idea of "notes" + "p
|
||||
**gbrain doesn't have a fixed layout.** It ships with bundled schema packs and lets you author your own when none fit:
|
||||
|
||||
- **`gbrain-base-v2`** (default as of v0.41.22) — 15-type DRY/MECE canonical taxonomy (14 canonical + `note` catch-all): `person`, `company`, `media`, `tweet`, `social-digest`, `analysis`, `atom`, `concept`, `source`, `deal`, `email`, `slack`, `writing`, `project`, `note`. Subtypes/format/origin pushed to frontmatter. The taxonomy that responds to issue #1479.
|
||||
- **`gbrain-base`** (legacy, v0.41 and earlier brains) — the original 24-type layout. Stays bundled for back-compat; brains on it can upgrade via `gbrain onboard --check --explain` → `gbrain jobs submit unify-types --allow-protected --params '{"target_pack":"gbrain-base-v2"}'`.
|
||||
- **`gbrain-base`** (legacy, v0.41 and earlier brains) — the original 24-type layout. Stays bundled for back-compat; brains on it can upgrade via `gbrain onboard --check --explain` → `gbrain jobs submit unify-types --allow-protected --params '{"target_pack":"gbrain-base-v2","apply":true}'` (omit `"apply":true` for a dry-run preview — that is the default).
|
||||
- **`gbrain-recommended`** — extends `gbrain-base` with the 13 additional directories from `docs/GBRAIN_RECOMMENDED_SCHEMA.md` (source, place, trip, conversation, personal, civic, project, etc.). Activate with `gbrain schema use gbrain-recommended`.
|
||||
- **Your own pack** — `gbrain schema detect` clusters your actual filesystem into proposed types, `gbrain schema suggest` runs an LLM pass over them, and `gbrain schema review-candidates --apply` promotes the ones you like. Three commands and the brain knows your shape. Authoring a successor pack (declares `migration_from:` so existing brains can opt in): see [`docs/architecture/pack-upgrade-mechanism.md`](docs/architecture/pack-upgrade-mechanism.md).
|
||||
|
||||
@@ -1754,6 +1784,24 @@ The whole loop is described in [`docs/architecture/topologies.md`](docs/architec
|
||||
|
||||
**Job queue (Minions).** BullMQ-shaped, Postgres-native job queue. Durable subagents (LLM tool loops that survive crashes via two-phase pending→done persistence), shell jobs with audit, child jobs with cascading timeouts, rate leases for outbound providers, attachments via S3/Supabase storage. Replaces "spawn subagent as fire-and-forget Promise" with something that recovers from anything.
|
||||
|
||||
**Non-English brains (FTS language config).** The Postgres full-text search tokenizer is configurable via `GBRAIN_FTS_LANGUAGE`. Defaults to `english`. Set it to any text-search configuration that exists in your Postgres instance:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese # uses built-in portuguese stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=spanish # built-in spanish stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br # custom config (e.g. unaccent + portuguese)
|
||||
```
|
||||
|
||||
List available configs: `psql -c "SELECT cfgname FROM pg_ts_config"`. Both the **query side** (`websearch_to_tsquery`) and the **write side** (the trigger functions that populate `pages.search_vector` and `content_chunks.search_vector`) honor `GBRAIN_FTS_LANGUAGE`. On first install (or upgrade), the `configurable_fts_language` schema migration reads the env var and creates trigger functions in the configured language; subsequent inserts/updates tokenize using that setting. To change language on a brain that has already run the migration, use the dedicated CLI command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command is idempotent (re-running with the same language is a no-op for vector content) and uses the same recreate-and-backfill primitives as the migration. For accent-insensitive Portuguese (`pt_br`), see [docs/guides/multi-language-fts.md](docs/guides/multi-language-fts.md) for the `unaccent` + portuguese stemmer recipe.
|
||||
|
||||
**43 curated skills.** Routing lives in [`skills/RESOLVER.md`](skills/RESOLVER.md). Covers signal capture, ingest (idea / media / meeting), enrichment, querying, brain ops, citation fixing, daily task management, cron scheduling, reports, voice, soul audit, skill creation, eval framework, and migrations. Skills are markdown files (tool-agnostic), packaged as a single skillpack the installer drops into your agent workspace.
|
||||
|
||||
**Eval framework.** `gbrain eval longmemeval` runs the public [LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval) benchmark against your hybrid retrieval. `gbrain eval export` + `gbrain eval replay` capture real queries and replay them against code changes (set `GBRAIN_CONTRIBUTOR_MODE=1`). `gbrain eval cross-modal` cross-checks an output against the task using three different-provider frontier models. `gbrain eval retrieval-quality` runs NamedThingBench, which hard-gates the named-thing retrieval families (title-substring, alias-synonym, generic-to-named, multi-chunk-dilution) so a regression in "find the page this query names" fails CI loudly. Full methodology in [`docs/eval/SEARCH_MODE_METHODOLOGY.md`](docs/eval/SEARCH_MODE_METHODOLOGY.md).
|
||||
@@ -1785,6 +1833,8 @@ Data flowing into the brain. Each integration is a recipe — markdown + setup h
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**`gbrain init --pglite` crashes on macOS 26.x (Tahoe)?** PGLite's embedded WASM engine is incompatible with macOS 26.x on Apple Silicon. The fix is to use native Homebrew PostgreSQL + pgvector instead. Full step-by-step setup in [`docs/INSTALL.md` — Troubleshooting: PGLite crashes on macOS 26.x](docs/INSTALL.md#pglite-crashes-on-macos-26x-tahoe).
|
||||
|
||||
**`gbrain import` fails with `expected N dimensions, not M`?** Run `gbrain doctor`. It will print the exact `gbrain config set ...` or `gbrain retrieval-upgrade` command to repair the mismatch. You should not need to delete `~/.gbrain`. Fresh `gbrain init --pglite` auto-detects your embedding provider from API keys in your environment: set `OPENAI_API_KEY` (or `ZEROENTROPY_API_KEY` / `VOYAGE_API_KEY`) before running init, or pass `--embedding-model <provider>:<model>` explicitly. With multiple keys set, init fires an interactive picker. In non-TTY contexts (CI, Docker) with no keys, init exits 1 with a paste-ready setup hint; pass `--no-embedding` to defer setup until runtime. See [`docs/integrations/embedding-providers.md`](docs/integrations/embedding-providers.md) for the full provider matrix and [`docs/operations/headless-install.md`](docs/operations/headless-install.md) for Docker/CI sequencing.
|
||||
|
||||
**Hourly cron sync keeps timing out on a federated brain?** v0.41.13.0 ships
|
||||
@@ -2095,6 +2145,51 @@ RRF fusion, multi-query expansion, and 4-layer dedup are engine-agnostic. They o
|
||||
|
||||
**Why not self-hosted for v0:** The brain should be infrastructure agents use, not something you maintain. Self-hosted Postgres with Docker is a welcome community PR, but v0 optimizes for zero ops.
|
||||
|
||||
### Opt-in RLS source-scope binding (`GBRAIN_RLS_SCOPE_BINDING`)
|
||||
|
||||
Defense-in-depth layer for Postgres deployments that want the database itself
|
||||
to enforce source isolation, in addition to the mandatory app-layer filters
|
||||
(`sourceScopeOpts` — layer 1, always on).
|
||||
|
||||
**Mechanism.** With `GBRAIN_RLS_SCOPE_BINDING=1` (or `true`), the engine's
|
||||
source-scoped read methods wrap their queries in a transaction that first runs
|
||||
`SELECT set_config('app.scopes', $1, true)` — the value is a bound parameter
|
||||
(federated `sourceIds` CSV > scalar `sourceId` > `'*'` for unscoped internal
|
||||
reads), transaction-local (equivalent to `SET LOCAL`, which itself can't take
|
||||
bound params). An RLS policy can then filter rows by
|
||||
`current_setting('app.scopes', true)`.
|
||||
|
||||
**Default off.** With the env var unset, reads call through on the shared pool
|
||||
exactly as before — no per-read transaction, no pool-slot hold (the search
|
||||
methods keep the transaction they always had for their `SET LOCAL
|
||||
statement_timeout`). Existing operators see zero behavior change.
|
||||
|
||||
**Enabling it** (operator-managed SQL; gbrain ships no DDL for this):
|
||||
|
||||
```sql
|
||||
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
|
||||
CREATE POLICY pages_scope_filter ON pages
|
||||
USING (current_setting('app.scopes', true) = '*'
|
||||
OR source_id = ANY(string_to_array(current_setting('app.scopes', true), ',')));
|
||||
|
||||
-- Required: connections that don't run through the scoped read helper
|
||||
-- (admin, autopilot, cycle, writes) must default to unscoped, or they
|
||||
-- see zero rows once the policy exists:
|
||||
ALTER ROLE <runtime-role> SET app.scopes = '*';
|
||||
|
||||
-- If the runtime role OWNS the table, RLS is skipped for it unless forced:
|
||||
ALTER TABLE pages FORCE ROW LEVEL SECURITY;
|
||||
```
|
||||
|
||||
Safe to enable in either order: the env var without a policy is a no-op
|
||||
setting; a policy without the env var is enforced only via the role default.
|
||||
|
||||
**Honest caveat:** only read paths routed through the scoped helper carry a
|
||||
per-request scope binding — unwrapped paths (writes, admin/maintenance reads)
|
||||
run under the role default and are not backstopped per caller. This is layer 2;
|
||||
the app-layer source filters remain layer 1 and stay mandatory. Behavioral pins
|
||||
live in `test/postgres-engine-rls-scope.test.ts`.
|
||||
|
||||
## PGLiteEngine (v0.7, ships)
|
||||
|
||||
**Dependencies:** `@electric-sql/pglite` (v0.4.4+)
|
||||
@@ -2253,7 +2348,7 @@ gbrain schema sync --apply
|
||||
The sync backfills `page.type = 'meeting'` on all 4000 pages in 1000-row batches. Now:
|
||||
|
||||
- `gbrain whoknows "Q3 roadmap discussion"` routes through the meeting type, ranking by `expert_routing` signal (attendees, recency, salience) instead of raw text.
|
||||
- `gbrain extract-facts` runs on every meeting page automatically (because `extractable: true`), pulling typed facts like `attended_by=alice-example`, `date=2026-05-23`.
|
||||
- The `extract_facts` cycle runs on every meeting page automatically (because `extractable: true`), pulling typed facts like `attended_by=alice-example`, `date=2026-05-23`.
|
||||
- The downstream `think` skill can now answer "what did we decide about pricing in the last three roadmap meetings" by querying the meeting graph instead of grep'ing 4000 files.
|
||||
|
||||
One command. 4000 pages went from invisible to queryable. The content didn't change. The structure did.
|
||||
@@ -2283,7 +2378,7 @@ gbrain schema add-link-type led-by --page-type deal --target-type inves
|
||||
gbrain schema sync --apply
|
||||
```
|
||||
|
||||
Now `gbrain whoknows "Series A SaaS"` routes through `investor` and `portco` types specifically, not the noisy general type set. `gbrain graph-query alice-example --type intro-from --depth 2` walks two hops of intros to surface "Alice introduced you to Bob who introduced you to Charlie." `gbrain extract-facts` starts producing typed claims from the fence in your deal pages: `(deals/acme-seed, raise=2000000, valuation=15000000, lead=widget-vc, closed_at=2026-05-23)`.
|
||||
Now `gbrain whoknows "Series A SaaS"` routes through `investor` and `portco` types specifically, not the noisy general type set. `gbrain graph-query alice-example --type intro-from --depth 2` walks two hops of intros to surface "Alice introduced you to Bob who introduced you to Charlie." The `extract_facts` cycle starts producing typed claims from the fence in your deal pages: `(deals/acme-seed, raise=2000000, valuation=15000000, lead=widget-vc, closed_at=2026-05-23)`.
|
||||
|
||||
The CRM you've been promising yourself you'll set up next quarter? You just shipped it in 4 commands. It's downstream of your notes, not parallel to them.
|
||||
|
||||
@@ -2364,7 +2459,7 @@ Re-run the same `whoknows` query. Top-3 should shift, because the new type is no
|
||||
|
||||
Three things gbrain does that generic note systems can't:
|
||||
|
||||
**1. The brain knows the difference between a person and an idea.** Page-type matters at query time. `gbrain whoknows` only considers `expert_routing: true` types. `gbrain extract-facts` only runs on `extractable: true` types. `gbrain graph-query` walks declared link verbs. None of that works on a flat tag system because tags don't have semantics — they're labels. Types are first-class citizens with rules attached.
|
||||
**1. The brain knows the difference between a person and an idea.** Page-type matters at query time. `gbrain whoknows` only considers `expert_routing: true` types. The `extract_facts` cycle only runs on `extractable: true` types. `gbrain graph-query` walks declared link verbs. None of that works on a flat tag system because tags don't have semantics — they're labels. Types are first-class citizens with rules attached.
|
||||
|
||||
**2. Untyped content is invisible content.** If your meetings are typed as `note`, expert routing skips them, facts extraction ignores them, link inference doesn't fire. They exist on disk and they're indexed for text search, but the structural surfaces (whoknows, find_experts, recall, think) treat them as second-class. Adding a type isn't cosmetic; it's structural promotion.
|
||||
|
||||
@@ -2657,14 +2752,17 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
@@ -2767,6 +2865,16 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict.
|
||||
history rewrite still hard-blocks even with `--skip-failed`. Run
|
||||
`gbrain sync --skip-failed` to acknowledge a known-bad set yourself.
|
||||
|
||||
5. **Import checkpoints name the import target, not the caller's CWD.**
|
||||
Interrupted `gbrain import <dir>` runs may leave
|
||||
`~/.gbrain/import-checkpoint.json` so the next import can resume. The
|
||||
checkpoint `dir` is the absolute, resolved import target captured when
|
||||
import starts. It is not a cleanup instruction and it must not be
|
||||
re-derived from the process working directory. Checkpoints written by
|
||||
gbrain include `schema_version: 1`, `owner: "gbrain"`, and
|
||||
`kind: "import"` so downstream tools can validate the contract before
|
||||
deciding whether to resume.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Edit a file and search for the change.** Edit a brain markdown file,
|
||||
@@ -3650,13 +3758,20 @@ to the HTTP server, so no migration is required.
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
On first start, the server prints an **admin bootstrap token** to stderr:
|
||||
On first start in an interactive terminal, the server prints an **admin
|
||||
bootstrap token** to stderr:
|
||||
|
||||
```
|
||||
Admin bootstrap token: 3a1f9c...
|
||||
Open http://localhost:3131/admin and paste it to log in.
|
||||
```
|
||||
|
||||
On a non-TTY start (systemd, Docker, any piped or captured logs) the generated
|
||||
token is hidden so it never lands in log storage. For headless deploys either
|
||||
set `GBRAIN_ADMIN_BOOTSTRAP_TOKEN` to a value you control before starting, or
|
||||
run `gbrain serve --http --print-admin-token` once on a trusted terminal to
|
||||
force printing.
|
||||
|
||||
Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
@@ -3819,7 +3934,7 @@ All 30 GBrain operations are available remotely, including `sync_brain` and
|
||||
directory where `gbrain serve` was launched. Symlinks, `..` traversal, and absolute
|
||||
paths outside cwd are rejected. Page slugs and filenames are allowlist-validated
|
||||
(alphanumeric + hyphens; no control chars, RTL overrides, or backslashes). Local
|
||||
CLI callers (`gbrain file upload ...`) keep unrestricted filesystem access since
|
||||
CLI callers (`gbrain files upload ...`) keep unrestricted filesystem access since
|
||||
the user owns the machine.
|
||||
|
||||
## Deployment Options
|
||||
@@ -3827,6 +3942,43 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.32.3.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
|
||||
+52
-36
@@ -23,6 +23,7 @@
|
||||
"./backoff": "./src/core/backoff.ts",
|
||||
"./search/hybrid": "./src/core/search/hybrid.ts",
|
||||
"./search/expansion": "./src/core/search/expansion.ts",
|
||||
"./think": "./src/core/think/index.ts",
|
||||
"./ai/gateway": "./src/core/ai/gateway.ts",
|
||||
"./extract": "./src/commands/extract.ts",
|
||||
"./ingestion": "./src/core/ingestion/index.ts",
|
||||
@@ -41,20 +42,21 @@
|
||||
"eval:autocut": "bun test test/search/autocut-eval.test.ts",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bash scripts/run-verify-parallel.sh",
|
||||
"check:source-config-leak": "scripts/check-source-config-leak.sh",
|
||||
"check:no-pii-agent-voice": "scripts/check-no-pii-in-agent-voice.sh",
|
||||
"check:synthetic-corpus-privacy": "scripts/check-synthetic-corpus-privacy.sh",
|
||||
"check:system-of-record": "scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "scripts/check-cli-executable.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-proposal-pii.sh && scripts/check-test-real-names.sh && scripts/check-jsonb-pattern.sh && scripts/check-source-id-projection.sh && scripts/check-source-config-leak.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh && scripts/check-skill-brain-first.sh && scripts/check-operations-filter-bypass.sh && scripts/check-gateway-routed-no-direct-anthropic.sh && scripts/check-worker-pool-atomicity.sh && scripts/check-key-files-current-state.sh && scripts/check-no-double-retry.sh && scripts/check-batch-audit-site.sh",
|
||||
"check:gateway-routed": "scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "scripts/check-key-files-current-state.sh",
|
||||
"check:source-config-leak": "bash scripts/check-source-config-leak.sh",
|
||||
"check:no-pii-agent-voice": "bash scripts/check-no-pii-in-agent-voice.sh",
|
||||
"check:synthetic-corpus-privacy": "bash scripts/check-synthetic-corpus-privacy.sh",
|
||||
"check:system-of-record": "bash scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "bash scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "bash scripts/check-cli-executable.sh",
|
||||
"check:engine-dynamic-import": "bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:all": "bash scripts/check-privacy.sh && bash scripts/check-proposal-pii.sh && bash scripts/check-test-real-names.sh && bash scripts/check-jsonb-pattern.sh && bash scripts/check-source-id-projection.sh && bash scripts/check-source-config-leak.sh && bash scripts/check-progress-to-stdout.sh && bash scripts/check-no-tracked-symlinks.sh && bash scripts/check-no-legacy-getconnection.sh && bash scripts/check-test-isolation.sh && bash scripts/check-trailing-newline.sh && bash scripts/check-wasm-embedded.sh && bash scripts/check-exports-count.sh && bash scripts/check-admin-build.sh && bash scripts/check-admin-scope-drift.sh && bash scripts/check-cli-executable.sh && bash scripts/check-skill-brain-first.sh && bash scripts/check-operations-filter-bypass.sh && bash scripts/check-gateway-routed-no-direct-anthropic.sh && bash scripts/check-worker-pool-atomicity.sh && bash scripts/check-key-files-current-state.sh && bash scripts/check-no-double-retry.sh && bash scripts/check-batch-audit-site.sh && bash scripts/check-engine-dynamic-import.sh",
|
||||
"check:gateway-routed": "bash scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "bash scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "bash scripts/check-key-files-current-state.sh",
|
||||
"check:resolver": "bun src/cli.ts check-resolvable --strict --skills-dir skills/",
|
||||
"check:skill-brain-first": "scripts/check-skill-brain-first.sh",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"check:skill-brain-first": "bash scripts/check-skill-brain-first.sh",
|
||||
"check:wasm": "bash scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "bash scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:heavy": "bash scripts/run-heavy.sh",
|
||||
@@ -64,27 +66,29 @@
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check:jsonb": "scripts/check-jsonb-pattern.sh",
|
||||
"check:search-path": "scripts/check-search-path.sh",
|
||||
"check:no-double-retry": "scripts/check-no-double-retry.sh",
|
||||
"check:batch-audit-site": "scripts/check-batch-audit-site.sh",
|
||||
"check:worker-lock-renewal-shape": "scripts/check-worker-lock-renewal-shape.sh",
|
||||
"check:source-id-projection": "scripts/check-source-id-projection.sh",
|
||||
"check:privacy": "scripts/check-privacy.sh",
|
||||
"check:proposal-pii": "scripts/check-proposal-pii.sh",
|
||||
"check:eval-glossary": "scripts/check-eval-glossary-fresh.sh",
|
||||
"check:test-names": "scripts/check-test-real-names.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "scripts/check-admin-embedded.sh",
|
||||
"check:test-isolation": "scripts/check-test-isolation.sh",
|
||||
"check:fuzz-purity": "scripts/check-fuzz-purity.sh",
|
||||
"check:operations-filter-bypass": "scripts/check-operations-filter-bypass.sh",
|
||||
"check:fixture-privacy": "scripts/check-fixture-privacy.sh",
|
||||
"check:jsonb": "bash scripts/check-jsonb-pattern.sh",
|
||||
"check:search-path": "bash scripts/check-search-path.sh",
|
||||
"check:no-double-retry": "bash scripts/check-no-double-retry.sh",
|
||||
"check:batch-audit-site": "bash scripts/check-batch-audit-site.sh",
|
||||
"check:worker-lock-renewal-shape": "bash scripts/check-worker-lock-renewal-shape.sh",
|
||||
"check:source-id-projection": "bash scripts/check-source-id-projection.sh",
|
||||
"check:privacy": "bash scripts/check-privacy.sh",
|
||||
"check:proposal-pii": "bash scripts/check-proposal-pii.sh",
|
||||
"check:eval-glossary": "bash scripts/check-eval-glossary-fresh.sh",
|
||||
"check:skills-manifest": "bash scripts/check-skills-manifest-fresh.sh",
|
||||
"check:test-names": "bash scripts/check-test-real-names.sh",
|
||||
"check:progress": "bash scripts/check-progress-to-stdout.sh",
|
||||
"check:no-tracked-symlinks": "bash scripts/check-no-tracked-symlinks.sh",
|
||||
"check:exports-count": "bash scripts/check-exports-count.sh",
|
||||
"check:admin-build": "bash scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "bash scripts/check-admin-embedded.sh",
|
||||
"check:test-isolation": "bash scripts/check-test-isolation.sh",
|
||||
"check:fuzz-purity": "bash scripts/check-fuzz-purity.sh",
|
||||
"check:operations-filter-bypass": "bash scripts/check-operations-filter-bypass.sh",
|
||||
"check:fixture-privacy": "bash scripts/check-fixture-privacy.sh",
|
||||
"check:conversation-parser": "bun src/cli.ts eval conversation-parser test/fixtures/conversation-formats/all.jsonl --no-llm",
|
||||
"check:source-scope-onboard": "scripts/check-source-scope-onboard.sh",
|
||||
"postinstall": "command -v gbrain >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive || echo '[gbrain] postinstall skipped. If installed via bun install -g github:...: run `gbrain doctor` and `gbrain apply-migrations --yes` manually. See https://github.com/garrytan/gbrain/issues/218' 1>&2",
|
||||
"check:source-scope-onboard": "bash scripts/check-source-scope-onboard.sh",
|
||||
"postinstall": "bun run scripts/postinstall.ts",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
},
|
||||
@@ -118,8 +122,8 @@
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"heic-decode": "^2.1.0",
|
||||
"js-yaml": "^3.14.2",
|
||||
"marked": "^18.0.0",
|
||||
"js-yaml": "^3.15.0",
|
||||
"marked": "^18.0.2",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
@@ -144,5 +148,17 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.43.0.0"
|
||||
"version": "0.43.0.0",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.5",
|
||||
"body-parser": "^2.3.0",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
"hono": "^4.12.34",
|
||||
"ip-address": "^10.3.1",
|
||||
"qs": "^6.15.2",
|
||||
"js-yaml": "^3.15.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: calendar-to-brain
|
||||
name: Calendar-to-Brain
|
||||
version: 0.7.0
|
||||
version: 0.8.0
|
||||
description: Google Calendar events become searchable brain pages. Daily files with attendees, locations, and meeting prep context.
|
||||
category: sense
|
||||
requires: [credential-gateway]
|
||||
@@ -28,6 +28,11 @@ health_checks:
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
label: "Google OAuth"
|
||||
- type: heartbeat_max_age
|
||||
max_age: 48h
|
||||
label: "Calendar data freshness"
|
||||
output_paths:
|
||||
- daily/calendar/
|
||||
setup_time: 20 min
|
||||
cost_estimate: "$0 (both options are free)"
|
||||
---
|
||||
|
||||
+25
-12
@@ -1,19 +1,22 @@
|
||||
---
|
||||
id: x-to-brain
|
||||
name: X-to-Brain
|
||||
version: 0.8.1
|
||||
version: 0.8.3
|
||||
description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts.
|
||||
category: sense
|
||||
requires: []
|
||||
secrets:
|
||||
- name: X_BEARER_TOKEN
|
||||
- name: X_API_BEARER_TOKEN
|
||||
description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search)
|
||||
where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens"
|
||||
- name: X_HANDLE
|
||||
description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have)
|
||||
where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle
|
||||
health_checks:
|
||||
- type: http
|
||||
url: "https://api.x.com/2/users/me"
|
||||
url: "https://api.x.com/2/users/by/username/$X_HANDLE"
|
||||
auth: bearer
|
||||
auth_token: "$X_BEARER_TOKEN"
|
||||
auth_token: "$X_API_BEARER_TOKEN"
|
||||
label: "X API"
|
||||
setup_time: 15 min
|
||||
cost_estimate: "$0-200/mo (Free tier: 1 app, read-only. Basic: $200/mo for search + higher limits)"
|
||||
@@ -110,15 +113,17 @@ Tell the user:
|
||||
4. Inside the project, create a new App
|
||||
5. Go to the app's 'Keys and tokens' tab
|
||||
6. Under 'Bearer Token', click 'Generate' (or 'Regenerate')
|
||||
7. Copy the Bearer Token and paste it to me
|
||||
7. Copy the Bearer Token and paste it to me, along with your X handle (without the @)
|
||||
|
||||
Note: Free tier gives read-only access with low limits. Basic tier ($200/mo)
|
||||
gives search/recent endpoint and higher limits. Pro tier gets full archive search."
|
||||
|
||||
Validate immediately:
|
||||
Set both `X_API_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
(app-only bearer tokens cannot call `/users/me` — that endpoint requires
|
||||
user-context OAuth — so validation uses the by-username lookup):
|
||||
```bash
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/me" \
|
||||
curl -sf -H "Authorization: Bearer $X_API_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" \
|
||||
&& echo "PASS: X API connected" \
|
||||
|| echo "FAIL: X API token invalid"
|
||||
```
|
||||
@@ -133,11 +138,11 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme
|
||||
|
||||
```bash
|
||||
# Look up the user's X user ID from their handle
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"'
|
||||
curl -sf -H "Authorization: Bearer $X_API_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"'
|
||||
```
|
||||
|
||||
Ask the user for their X handle (e.g., @yourhandle). Look up their user ID.
|
||||
Look up the user ID from the handle collected in Step 1.
|
||||
Save it — the collector needs the numeric ID, not the handle.
|
||||
|
||||
### Step 3: Configure the Collector
|
||||
@@ -205,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/x-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.3","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
## Production Patterns (v0.8.1)
|
||||
@@ -433,6 +438,14 @@ Free tier works for personal monitoring. Basic tier needed for keyword search.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Upgrading from recipe v0.8.2 or earlier (token shows [missing] after upgrade):**
|
||||
- Older versions of this recipe named the token `X_BEARER_TOKEN`. The canonical
|
||||
name is `X_API_BEARER_TOKEN` — the name the built-in `x_handle_to_tweet`
|
||||
resolver reads. Rename the variable wherever you set it (shell profile, cron
|
||||
environment, `.env`) — same value, new name. A collector installed under the
|
||||
old name keeps running either way; the rename is what makes the integrations
|
||||
dashboard and the resolver see the token.
|
||||
|
||||
**API returns 403:**
|
||||
- Check your app has the right access level (Read or Read+Write)
|
||||
- Free tier apps can only use basic endpoints
|
||||
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Print the CHANGELOG.md body for one version (Keep-a-Changelog format),
|
||||
# without its `## [X.Y.Z.W] - date` header line. Used by
|
||||
# .github/workflows/release.yml as the GitHub release notes; tested by
|
||||
# test/release-workflow.test.ts.
|
||||
#
|
||||
# Usage: changelog-entry.sh <version> [changelog-file]
|
||||
# Exits 1 when the version has no entry (caller falls back to a link stub).
|
||||
set -euo pipefail
|
||||
|
||||
ver="${1:?usage: changelog-entry.sh <version> [changelog-file]}"
|
||||
file="${2:-CHANGELOG.md}"
|
||||
|
||||
# Exact-string prefix match on "## [<ver>]" — no regex, so dots in the
|
||||
# version can't glob and a 3-segment version can't match a 4-segment header.
|
||||
awk -v ver="$ver" '
|
||||
index($0, "## [" ver "]") == 1 { found = 1; next }
|
||||
found && /^## \[/ { exit }
|
||||
found { print }
|
||||
END { exit found ? 0 : 1 }
|
||||
' "$file"
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: every `bun test` invocation in workflows and runner scripts must
|
||||
# pass an explicit --timeout.
|
||||
#
|
||||
# Why: bun ignores bunfig.toml's `timeout` key (verified on 1.3.14), so a bare
|
||||
# `bun test` gets the 5000ms default for BOTH tests and beforeAll/beforeEach/
|
||||
# afterAll/afterEach hooks. Hooks do NOT inherit a test's third-arg timeout —
|
||||
# a file whose tests all declare `}, 30_000)` still has a 5s hook budget, and
|
||||
# slow setup (Postgres connect + migrations, PGLite cold start) flakes on
|
||||
# loaded CI runners with the signature `(unnamed) [5001ms] ... hook timed out`
|
||||
# (the #3545 jsonb-parity failure). The CLI --timeout flag is the one measured
|
||||
# mechanism that raises the hook budget uniformly; per-hook second-arg
|
||||
# timeouts work too but don't scale to ~400 slow hooks.
|
||||
#
|
||||
# Usage: scripts/check-bun-test-timeout.sh
|
||||
# Exit: 0 when clean, 1 when a bare `bun test` invocation is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Match executable `bun test` invocations. Exclude comment lines (#, //, *)
|
||||
# and lines that already carry --timeout anywhere.
|
||||
# Scope: workflows + runner scripts (the surfaces CI executes). package.json
|
||||
# script bodies route through scripts/ already; editing it is out of scope here.
|
||||
violations="$(grep -rnE '\bbun test\b' .github/workflows scripts 2>/dev/null \
|
||||
| grep -v -- '--timeout' \
|
||||
| grep -vE ':[[:space:]]*(#|//|\*)' \
|
||||
| grep -v 'check-bun-test-timeout' \
|
||||
|| true)"
|
||||
|
||||
if [ -n "$violations" ]; then
|
||||
echo "FAIL: bare 'bun test' without --timeout (5s default kills slow setup hooks):" >&2
|
||||
echo "$violations" >&2
|
||||
echo "" >&2
|
||||
echo "Add --timeout=60000 (see scripts/run-unit-shard.sh for the convention)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: every bun test invocation passes an explicit --timeout."
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Engine-live paths use static imports by default. A line-level
|
||||
# `engine-dynamic-import-ok` marker is required for a justified lazy import.
|
||||
#
|
||||
# Historical Windows runs associated imports on these paths with abrupt Bun
|
||||
# test-process exits, but system-wide commit exhaustion remained a confound.
|
||||
# This guard therefore enforces a reviewed engine-path hardening invariant; it
|
||||
# does not claim every dynamic import deterministically crashes Windows.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/check-engine-dynamic-import.sh
|
||||
# bash scripts/check-engine-dynamic-import.sh FILE [FILE...]
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" || exit 1
|
||||
|
||||
if [ "$#" -gt 0 ]; then
|
||||
FILES=("$@")
|
||||
else
|
||||
ROOT="$(git -C "$SCRIPT_DIR/.." rev-parse --show-toplevel 2>/dev/null || true)"
|
||||
[ -n "$ROOT" ] || ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$ROOT" || exit 1
|
||||
FILES=(
|
||||
src/core/pglite-engine.ts
|
||||
src/core/postgres-engine.ts
|
||||
src/core/migrate.ts
|
||||
)
|
||||
fi
|
||||
|
||||
exec bun "$SCRIPT_DIR/check-engine-dynamic-import.ts" "${FILES[@]}"
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import ts from 'typescript';
|
||||
|
||||
const MARKER = 'engine-dynamic-import-ok';
|
||||
const MARKER_TOKEN_CHAR = /[\p{ID_Continue}$-]/u;
|
||||
const files = process.argv.slice(2);
|
||||
const violations: string[] = [];
|
||||
const readErrors: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
let sourceText: string;
|
||||
try {
|
||||
sourceText = await readFile(file, 'utf8');
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
readErrors.push(`ERROR: cannot read input file ${file}: ${detail}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceFile = ts.createSourceFile(
|
||||
file,
|
||||
sourceText,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
const lines = sourceText.split(/\r?\n/);
|
||||
const markerLines = new Set<number>();
|
||||
|
||||
if (sourceFile.parseDiagnostics.length > 0) {
|
||||
const diagnostics = sourceFile.parseDiagnostics
|
||||
.map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, ' '))
|
||||
.join('; ');
|
||||
readErrors.push(`ERROR: cannot parse input file ${file}: ${diagnostics}`);
|
||||
}
|
||||
|
||||
for (let markerPos = sourceText.indexOf(MARKER); markerPos >= 0; markerPos = sourceText.indexOf(MARKER, markerPos + MARKER.length)) {
|
||||
const before = Array.from(sourceText.slice(0, markerPos)).at(-1);
|
||||
const after = Array.from(sourceText.slice(markerPos + MARKER.length))[0];
|
||||
const standaloneMarker = (!before || !MARKER_TOKEN_CHAR.test(before))
|
||||
&& (!after || !MARKER_TOKEN_CHAR.test(after));
|
||||
const token = ts.getTokenAtPosition(sourceFile, markerPos);
|
||||
const insideToken = token.getStart(sourceFile) <= markerPos && markerPos < token.end;
|
||||
if (standaloneMarker && !insideToken) {
|
||||
markerLines.add(sourceFile.getLineAndCharacterOfPosition(markerPos).line);
|
||||
}
|
||||
}
|
||||
|
||||
function visit(node: ts.Node): void {
|
||||
if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) {
|
||||
const { line } = sourceFile.getLineAndCharacterOfPosition(node.expression.getStart(sourceFile));
|
||||
const sourceLine = lines[line] ?? '';
|
||||
if (!markerLines.has(line)) {
|
||||
violations.push(` ${file}:${line + 1}:${sourceLine}`);
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
|
||||
visit(sourceFile);
|
||||
}
|
||||
|
||||
for (const error of readErrors) console.error(error);
|
||||
|
||||
if (violations.length > 0) {
|
||||
console.error('ERROR: unreviewed dynamic import on an engine-live path:');
|
||||
console.error();
|
||||
console.error(violations.join('\n'));
|
||||
console.error();
|
||||
console.error('Prefer a static top-level import. If lazy loading is load-bearing,');
|
||||
console.error("append 'engine-dynamic-import-ok' to that exact line and document");
|
||||
console.error('the startup or soft-failure boundary that requires it.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (readErrors.length > 0) process.exit(1);
|
||||
|
||||
console.log(`check-engine-dynamic-import: ok (${files.length} file(s) scanned)`);
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_COUNT=20
|
||||
EXPECTED_COUNT=21
|
||||
|
||||
# Count top-level keys in the exports object. `node -e` parses JSON
|
||||
# reliably without needing jq (which isn't in every CI environment).
|
||||
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any symlink is tracked in git.
|
||||
#
|
||||
# A symlink committed from a build sandbox points at a path that exists on
|
||||
# exactly one machine. Everywhere else the checkout produces a dangling
|
||||
# link, and anything that opens it fails. That is not hypothetical: commit
|
||||
# faf5cdba landed `node_modules -> /tmp/fleet/repo/node_modules`, which made
|
||||
# `bun install` abort with `ENOENT: could not open the "node_modules"
|
||||
# directory` on every fresh clone, and took `gbrain upgrade`'s bun-link path
|
||||
# down with it (the auto-upgrade runs `bun install`, so the printed manual
|
||||
# fallback failed the same way).
|
||||
#
|
||||
# .gitignore alone does not prevent this. A `node_modules/` pattern with a
|
||||
# trailing slash matches directories ONLY, so a symlink of the same name is
|
||||
# never ignored. Dropping the slash closes that hole, but `git add -f` still
|
||||
# walks straight past it. This guard is the backstop.
|
||||
#
|
||||
# The repo has no legitimate tracked symlinks, so the allowlist starts
|
||||
# empty. If you ever need one, add its exact repo-relative path to ALLOWLIST
|
||||
# below and explain why — a relative link that resolves inside the repo is
|
||||
# defensible; an absolute one almost never is.
|
||||
#
|
||||
# Usage: scripts/check-no-tracked-symlinks.sh
|
||||
# Exit: 0 when clean, 1 when a tracked symlink is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Paths permitted to be tracked symlinks. Empty by design.
|
||||
ALLOWLIST=()
|
||||
|
||||
# Git records symlinks with mode 120000. Field 4 of `ls-files -s` is the path
|
||||
# (tab-separated from the stage number), so cut on the tab to keep paths with
|
||||
# spaces intact.
|
||||
found="$(git ls-files -s | awk '$1 == "120000"' | cut -f2- || true)"
|
||||
|
||||
if [ -n "$found" ]; then
|
||||
filtered="$found"
|
||||
for f in "${ALLOWLIST[@]:-}"; do
|
||||
[ -z "$f" ] && continue
|
||||
filtered="$(echo "$filtered" | grep -vxF "$f" || true)"
|
||||
done
|
||||
|
||||
if [ -n "$filtered" ]; then
|
||||
echo "ERROR: symlink(s) tracked in git:"
|
||||
echo
|
||||
while IFS= read -r path; do
|
||||
[ -z "$path" ] && continue
|
||||
target="$(git cat-file blob ":$path" 2>/dev/null || echo '<unreadable>')"
|
||||
echo " $path -> $target"
|
||||
done <<< "$filtered"
|
||||
echo
|
||||
echo "A committed symlink resolves on the machine that created it and"
|
||||
echo "nowhere else. Untrack it:"
|
||||
echo
|
||||
echo " git rm --cached <path>"
|
||||
echo
|
||||
echo "If the path is build output (node_modules, dist, bin), also confirm"
|
||||
echo "it is covered by .gitignore WITHOUT a trailing slash — a trailing"
|
||||
echo "slash matches directories only and lets the symlink through."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "check-no-tracked-symlinks: OK (no tracked symlinks)"
|
||||
@@ -70,7 +70,7 @@ PATTERN='import[[:space:]]+(\*[[:space:]]+as[[:space:]]+[a-zA-Z_$][a-zA-Z0-9_$]*
|
||||
FOUND_FILES=""
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] && FOUND_FILES="$FOUND_FILES$f"$'\n'
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src/ 2>/dev/null | sort -u || true)
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src 2>/dev/null | sort -u || true)
|
||||
|
||||
FAIL=0
|
||||
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard for skills/skills.lock.json freshness (#159).
|
||||
#
|
||||
# Mirrors scripts/check-eval-glossary-fresh.sh: regenerate the manifest into
|
||||
# a tmp file, diff against the committed version, fail the build if they
|
||||
# drift. Tamper-evidence, not a signature system — the point is that any
|
||||
# change under skills/ ships with an explicit manifest diff.
|
||||
#
|
||||
# Run: bash scripts/check-skills-manifest-fresh.sh
|
||||
# Wired through `bun run verify` (scripts/run-verify-parallel.sh) so PRs that
|
||||
# edit skills/ without regenerating the manifest are caught before review.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
COMMITTED="$REPO_ROOT/skills/skills.lock.json"
|
||||
TMP="$(mktemp)"
|
||||
trap 'rm -f "$TMP"' EXIT
|
||||
|
||||
if [ ! -f "$COMMITTED" ]; then
|
||||
echo "ERROR: $COMMITTED not found." >&2
|
||||
echo "Run: bun run scripts/generate-skills-manifest.ts" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
# Render directly via bun + a one-liner that exposes the module function.
|
||||
bun -e "import { renderSkillsManifest } from './src/core/skills-integrity.ts'; process.stdout.write(renderSkillsManifest('skills'));" > "$TMP"
|
||||
|
||||
if ! diff -q "$COMMITTED" "$TMP" >/dev/null 2>&1; then
|
||||
echo "ERROR: skills/skills.lock.json is stale." >&2
|
||||
echo "" >&2
|
||||
echo "Diff between committed and freshly-generated:" >&2
|
||||
echo "" >&2
|
||||
diff -u "$COMMITTED" "$TMP" >&2 || true
|
||||
echo "" >&2
|
||||
echo "To regenerate: bun run scripts/generate-skills-manifest.ts" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ skills/skills.lock.json is fresh"
|
||||
@@ -100,9 +100,9 @@ IFS='|' eval 'PATTERN="${PATTERN_PARTS[*]}"'
|
||||
|
||||
# Find tool.
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test/ 2>/dev/null || true)"
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test 2>/dev/null || true)"
|
||||
elif command -v grep >/dev/null 2>&1; then
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test/ 2>/dev/null || true)"
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test 2>/dev/null || true)"
|
||||
else
|
||||
echo "check-test-real-names: ERROR: neither rg nor grep available." >&2
|
||||
exit 2
|
||||
|
||||
@@ -19,13 +19,25 @@ set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
OUT_BIN="$(mktemp /tmp/gbrain-wasm-check.XXXXXX)"
|
||||
trap 'rm -f "$OUT_BIN"' EXIT
|
||||
# Build from a container-local copy. On Docker Desktop, Bun canonicalizes a
|
||||
# bind-mounted input to /run/host_virtiofs but keeps /app as the output path;
|
||||
# its final atomic rename then fails with ENOENT even though both names refer
|
||||
# to the same mount. Keeping inputs and output under /tmp avoids that alias.
|
||||
BUILD_DIR="$(mktemp -d /tmp/gbrain-wasm-check.XXXXXX)"
|
||||
OUT_BIN="$BUILD_DIR/chunker-smoketest"
|
||||
trap 'rm -rf "$BUILD_DIR"' EXIT
|
||||
mkdir -p "$BUILD_DIR/scripts"
|
||||
cp -R "$REPO_ROOT/src" "$BUILD_DIR/src"
|
||||
cp "$REPO_ROOT/scripts/chunker-smoketest.ts" "$BUILD_DIR/scripts/chunker-smoketest.ts"
|
||||
ln -s "$REPO_ROOT/node_modules" "$BUILD_DIR/node_modules"
|
||||
|
||||
# Build a minimal smoketest binary that imports the chunker. We compile this
|
||||
# instead of the full gbrain CLI so the failure mode is laser-focused on
|
||||
# chunker + WASM path resolution, not unrelated CLI wiring.
|
||||
bun build --compile --outfile "$OUT_BIN" scripts/chunker-smoketest.ts >/dev/null 2>&1
|
||||
if ! (cd "$BUILD_DIR" && bun build --compile --outfile "$OUT_BIN" scripts/chunker-smoketest.ts >/dev/null); then
|
||||
echo "[check-wasm-embedded] FAIL: bun could not compile the smoketest binary." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run it and capture JSON output.
|
||||
OUTPUT="$("$OUT_BIN" 2>&1)"
|
||||
|
||||
+1
-1
@@ -350,7 +350,7 @@ if [ -f .git ]; then
|
||||
fi
|
||||
|
||||
echo "[ci-local] Running checks inside runner container..."
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]:-}" runner bash -c "$INNER_CMD"
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]}" runner bash -c "$INNER_CMD"
|
||||
|
||||
echo ""
|
||||
echo "[ci-local] All checks passed."
|
||||
|
||||
+15
-2
@@ -42,8 +42,19 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
// phase, extract, integrity, embed, or migrate-engine change.
|
||||
"src/core/cycle/extract-takes.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/patterns.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/synthesize.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/embed.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/synthesize.ts": [
|
||||
"test/e2e/multi-source-bug-class.test.ts",
|
||||
"test/e2e/synthesize-bigint-job-id-postgres.test.ts",
|
||||
],
|
||||
"src/commands/embed.ts": [
|
||||
"test/e2e/multi-source-bug-class.test.ts",
|
||||
// #3391: the NULL-signature stale predicates differ per engine.
|
||||
"test/e2e/migrate-embeddings-postgres.test.ts",
|
||||
],
|
||||
// #3390: runSchemaTransition's DDL path + the stale predicates behave
|
||||
// differently on real pgvector than on PGLite.
|
||||
"src/core/embedding-migration.ts": ["test/e2e/migrate-embeddings-postgres.test.ts"],
|
||||
"src/core/retrieval-upgrade-planner.ts": ["test/e2e/migrate-embeddings-postgres.test.ts"],
|
||||
"src/commands/extract.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/migrate-engine.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
// Any minions queue/worker/handler change exercises all minion E2E.
|
||||
@@ -61,6 +72,8 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
"test/e2e/jsonb-roundtrip.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
// #3391: includeNullSignature stale predicates (engine parity).
|
||||
"test/e2e/migrate-embeddings-postgres.test.ts",
|
||||
],
|
||||
// PGLite bootstrap path + parity guard.
|
||||
"src/core/pglite-engine.ts": [
|
||||
|
||||
@@ -0,0 +1,981 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Import an envelope-v0 file (a JSON serialization of AI chat history; format
|
||||
* spec: github.com/memvelope/memvelope) into a brain repo as one Markdown page
|
||||
* per conversation, which `gbrain sync` ingests.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/envelope-to-gbrain.mjs <envelope.mve.json> [outDir]
|
||||
*
|
||||
* Zero dependencies. Deterministic. No network. It does NOT call gbrain — it
|
||||
* only writes Markdown files.
|
||||
*
|
||||
* All-or-nothing. Both integrity checks below run BEFORE the first write, so a
|
||||
* refused import leaves no partial output behind to be mistaken for a whole one.
|
||||
*
|
||||
* 1. Declared counts. envelope-v0 requires `meta.conversation_count` and
|
||||
* `meta.message_count`: the envelope states its own totals. Each is judged
|
||||
* on its own. One that disagrees with what the file actually contains, or
|
||||
* that is present but is not a non-negative integer, refuses the import
|
||||
* (exit 2) — a mismatch means the envelope is truncated, hand-edited, or
|
||||
* from a broken converter, and nothing here can tell which part is
|
||||
* missing. A count that is simply absent cannot be checked against
|
||||
* anything; the envelope imports, and stderr names the field whose half of
|
||||
* the check was skipped.
|
||||
* 2. Existing target files. A file already occupying a target filename is
|
||||
* only overwritten when it is safe: byte-identical content (a re-import),
|
||||
* or a page this importer wrote from the SAME conversation id (a refreshed
|
||||
* export legitimately updating its own page). Anything else — a foreign
|
||||
* file, or one of our pages whose conversation id cannot be matched — is a
|
||||
* conflict, and the import is refused (exit 2).
|
||||
*
|
||||
* Output layout:
|
||||
* - One page per conversation, filename = date + conversation id, so shared
|
||||
* titles cannot collide. The filename is that PAIR: a duplicated id whose
|
||||
* two copies carry different `created_at` DATES lands on two files and
|
||||
* nothing collides. A DUPLICATE ID IS NOT THE ONLY WAY TO REACH ONE
|
||||
* FILENAME, though: both halves are slugged — lowercased, every
|
||||
* non-alphanumeric run collapsed to a single `-`, leading and trailing `-`
|
||||
* stripped, and only THEN truncated at 60 characters — so two DISTINCT ids
|
||||
* can map to one name whenever their SLUGS agree on the first 60
|
||||
* characters. Say it on the slug and not on the id, because the raw ids
|
||||
* predict nothing in either direction: `AbC-123` and `abc-123` differ at
|
||||
* character 1 and collide, `x_y` and `x-y` differ at character 2 and
|
||||
* collide, while sixty `-` followed by `a` and sixty `-` followed by `b`
|
||||
* agree on all of their first 60 characters and do NOT collide (the strip
|
||||
* leaves `a` and `b`). This is the same class check 2 already names below —
|
||||
* "truncation at 60 chars, or characters that slug away" — except that
|
||||
* check 2 REFUSES it at exit 2 while the tiebreak below resolves it,
|
||||
* discarding one of two UNRELATED conversations while stderr calls the id
|
||||
* "not unique" and asks for a deduplication that cannot be performed. Real
|
||||
* ChatGPT and Claude exports carry lowercase UUIDs, which `slug` passes
|
||||
* through unchanged, so this is hand-authored-envelope territory rather
|
||||
* than producer output — but that is an observation about vendor data, not
|
||||
* a guarantee: the converter copies `raw.uuid` / `raw.conversation_id`
|
||||
* verbatim and validates nothing, and nothing here distinguishes the two
|
||||
* while resolving a collision.
|
||||
* When two conversations do map to one filename, the copy with the later
|
||||
* `updated_at` is kept and the other is discarded. That rule needs an
|
||||
* orderable `updated_at` on BOTH copies naming two different INSTANTS;
|
||||
* equal instants — which includes two different STRINGS that name one
|
||||
* instant, such as `09:00:00Z` and `14:30:00+05:30` — or a value that is
|
||||
* missing or unreadable on EITHER side, fall back to array order — the
|
||||
* later copy in `conversations[]` wins, as it always did. Either way
|
||||
* stderr names both values and which copy went, and stdout reports
|
||||
* DISTINCT files written, not write calls.
|
||||
* - `id` is `string | null` in envelope-v0 and a converter must not synthesize
|
||||
* one, so null is a conforming shape, not malformed input. Such a
|
||||
* conversation falls back to a POSITIONAL filename (`conv-N`) — a function
|
||||
* of array position, not of identity. That is precisely why check 2 refuses
|
||||
* to overwrite an id-less page: two unrelated exports both put their first
|
||||
* conversation at `conv-1`, and nothing in either file can distinguish
|
||||
* "this conversation, updated" from "a different conversation entirely".
|
||||
* BE PLAIN THAT THE TWO CHECKS DISAGREE HERE: within ONE envelope that name
|
||||
* is not refused but resolved — a positional `conv-1` and any real id that
|
||||
* SLUGS to `conv-1` share a filename, the `updated_at` tiebreak above picks
|
||||
* between them, and stderr reports a duplicate id where one of the two
|
||||
* conversations has no id at all (with the id-less copy second, it prints
|
||||
* `conversation id null is not unique`). It is the same conflation check 2
|
||||
* exists to forbid — two unrelated conversations resolved against one
|
||||
* positional name — though here it is loud and evidence-bearing, with
|
||||
* `updated_at` present on both copies and three stderr lines, rather than
|
||||
* the evidence-free overwrite check 2 refuses at exit 2. Note also that
|
||||
* `id: null` is not the only way INTO the positional namespace: `slug`
|
||||
* falls back to `conv-N` for any id that slugs to empty (`"___"`), and such
|
||||
* a page records that non-null id in frontmatter, so check 2 sees it as an
|
||||
* identity mismatch rather than as an id-less page. None of these are
|
||||
* producer-reachable — a vendor id of `conv-1` or `___` is not — but
|
||||
* `id: null` is.
|
||||
* - Frontmatter: `type: conversation` (keeps pages eligible for
|
||||
* conversation-facts extraction and chronicle behavior after sync), the
|
||||
* source provider, the conversation id, `origin: memvelope/envelope-v0`,
|
||||
* and the `messages:` array described below.
|
||||
* - Page `date` is the first 10 chars of the conversation's ISO-8601
|
||||
* `created_at`.
|
||||
*
|
||||
* THE BODY IS WRITTEN FOR GBRAIN'S OWN CONVERSATION PARSER.
|
||||
*
|
||||
* Every page here declares `type: conversation`, which is what opens the gate to
|
||||
* conversation-facts extraction, chronicle eligibility, and the
|
||||
* conversation_format_coverage check.
|
||||
*
|
||||
* ELIGIBLE IS NOT AUTOMATIC, and the difference is the whole reason to say this
|
||||
* out loud. The path that accepts these pages is `gbrain
|
||||
* extract-conversation-facts` (src/commands/extract-conversation-facts.ts) — a
|
||||
* command somebody starts, whether by hand, as a background job, or through a
|
||||
* `doctor` remediation. Its autopilot wrapper is the
|
||||
* `conversation_facts_backfill` cycle phase, and that phase is opt-in and OFF
|
||||
* by default (`cycle.conversation_facts_backfill.enabled`, default false —
|
||||
* src/core/cycle/conversation-facts-backfill.ts). A plain `gbrain sync` does
|
||||
* not start either one.
|
||||
*
|
||||
* What the type buys is ADMISSION to that command, not a trigger for it, and it
|
||||
* is one admission among several: `ALLOWED_TYPES` there is `conversation`,
|
||||
* `meeting`, `slack`, `email`, `imessage`, `imessage-daily`, and the command
|
||||
* defaults to the whole list. A page typed outside that set is ineligible
|
||||
* rather than merely un-run — which is the reason to declare `conversation`
|
||||
* here — but `conversation` is not privileged within it.
|
||||
*
|
||||
* Sync's own generic facts backstop is a SEPARATE gate and it does not accept
|
||||
* these pages on the type at all: `conversation` is absent from `ELIGIBLE_TYPES`
|
||||
* in src/core/facts/eligibility.ts. It has a slug escape hatch ORed with the
|
||||
* type test — `RESCUE_SLUG_PREFIXES = ['meetings/', 'personal/', 'daily/']` —
|
||||
* so a page written into an outDir that syncs under one of those prefixes IS
|
||||
* picked up by a plain sync, provided its body clears the 80-character
|
||||
* `MIN_BODY_CHARS` floor. The default outDir (`./brain/conversations`) is not
|
||||
* one of them, so on the default path nothing extracts facts from these pages
|
||||
* until the command above runs.
|
||||
*
|
||||
* Until 2026-08-02 the body then presented a
|
||||
* turn header — `**Assistant** (2025-11-02T14:22:51.000Z · m2):` — matching NONE
|
||||
* of the 17 built-in patterns in `src/core/conversation-parser/builtins.ts`
|
||||
* (`gbrain conversation-parser list-builtins` counts them). The
|
||||
* extractor parsed zero messages, incremented `pages_skipped`, and said nothing:
|
||||
* pages stored and searchable, no facts ever extracted from any of them.
|
||||
*
|
||||
* The header is now the one shape that parser reads:
|
||||
*
|
||||
* **Me** (2025-11-02 14:22):
|
||||
*
|
||||
* message text, on the following lines
|
||||
*
|
||||
* matching the `imessage-slack` built-in. That pattern's regex accepts
|
||||
* `YYYY-MM-DD` plus `H:MM` and an OPTIONAL AM/PM — a full RFC 3339 timestamp
|
||||
* does not match it (the `T` alone is enough to miss), and neither does anything
|
||||
* appended after the time. So the header can carry a wall clock and nothing
|
||||
* else, and per-message identity has to live in frontmatter:
|
||||
*
|
||||
* messages:
|
||||
* - id: "m1"
|
||||
* ts: "2025-11-02T14:22:51.000Z"
|
||||
* - id: "m2"
|
||||
* ts: "2025-11-02T14:24:03.000Z"
|
||||
*
|
||||
* TO READ IDENTITY BACK, a consumer parses the page's YAML frontmatter and
|
||||
* indexes `messages` BY POSITION: `messages[i]` is the i-th turn of the body, in
|
||||
* body order. There is no id in the body to join on. Both fields are copied from
|
||||
* the envelope verbatim — `id` is the message id, `ts` the original RFC 3339
|
||||
* timestamp (or `null`, which envelope-v0 permits). The body header is derived
|
||||
* FROM `ts` and is lossier than it by construction: minute resolution, UTC, and
|
||||
* a fallback whenever `ts` is null OR is a string this script will not read a
|
||||
* clock out of — a date with no time, a basic-format `20251102T142251Z`, an
|
||||
* impossible `2025-02-30`, anything non-string. `ts` is the record; the header
|
||||
* is the anchor, and only the record is lossless.
|
||||
*
|
||||
* Worth being plain about how much the array rescues: for a CONFORMING envelope
|
||||
* `id` is positional by spec (`m1`, `m2`, … restarting per conversation), so it
|
||||
* is derivable from the index and carries no information the position does not.
|
||||
* `ts` is the genuinely new value here. The `id` is recorded anyway because the
|
||||
* spec is what makes it derivable, and a non-conforming or future producer is
|
||||
* not bound by it.
|
||||
*
|
||||
* Every value TAKEN FROM THE ENVELOPE is JSON-encoded, so every timestamp
|
||||
* envelope-v0 can carry — `string | null` — is QUOTED or the bare `null`. (The
|
||||
* handful of fixed keys this script writes itself — `type: conversation`,
|
||||
* `origin:`, an absent `date: null`, an empty `messages: []` — are literals
|
||||
* under its own control, not envelope data.) Unquoted, an RFC 3339
|
||||
* scalar is read by js-yaml as a JS `Date`: microseconds truncate, a `+05:30`
|
||||
* offset is normalised away, the lexical form changes — and gbrain's own
|
||||
* `coerceFrontmatterString` (src/core/markdown.ts) slices a Date to its first
|
||||
* 10 characters, losing the time of day entirely. It is sticky, too: a Date
|
||||
* re-serializes unquoted and stays a Date on every later round trip.
|
||||
* `test/envelope-to-gbrain.test.ts` fails if a timestamp is ever emitted
|
||||
* unquoted, and carries a sentinel proving that guard fires.
|
||||
*
|
||||
* The precise claim, because "everything is quoted" would be false: JSON
|
||||
* quotes STRINGS. A non-conforming envelope whose `ts` is a number emits
|
||||
* `ts: 1762093371000` — unquoted, and a YAML integer rather than a Date, so it
|
||||
* is lossless and carries no `Date` hazard, but it is not a quoted scalar
|
||||
* either. Same for a non-string `id`, and a missing `id` (the schema requires
|
||||
* one) emits `id: null`, which is indistinguishable from a legitimate
|
||||
* `ts: null`. Coercing non-conforming types is deliberately not attempted here;
|
||||
* the behavior is pinned by test so it cannot drift unnoticed.
|
||||
*
|
||||
* The array survives a gbrain rewrite SEMANTICALLY, not textually.
|
||||
* `serializeMarkdown` re-emits `id: "m1"` as `id: m1` and `ts: "…"` as
|
||||
* `ts: '…'` — values and order identical, quoting style not. Anything that
|
||||
* reads this page by parsing YAML is fine; anything that reads it by scanning
|
||||
* lines must not assume double quotes.
|
||||
*
|
||||
* 24-hour, not 12-hour-with-AM/PM. Both match `imessage-slack`, and both were
|
||||
* measured to reconstruct all 24 hours exactly, so the tie is broken elsewhere:
|
||||
* 24-hour is a substring of the envelope's own `ts` (no hour arithmetic, so the
|
||||
* 12/0 boundary cannot be got wrong), it sorts chronologically within a day
|
||||
* where 12-hour does not, and it needs no AM/PM marker to disambiguate. The
|
||||
* pattern's `time_format: '12h_ampm'` declaration is not a constraint here.
|
||||
* Outside builtins.ts it is read in exactly one place — `list-builtins` prints
|
||||
* it (src/commands/conversation-parser.ts) — and never by the parser: parse.ts
|
||||
* converts off the CAPTURED AM/PM group, which is optional and absent for a
|
||||
* 24-hour clock, so `to24h(hour, undefined)` returns the hour unchanged.
|
||||
*
|
||||
* The stdout receipt reports MESSAGES as well as pages. Counting only pages hid
|
||||
* every message-level loss by construction: a conversation that arrives with
|
||||
* one turn instead of forty still writes exactly one page.
|
||||
*
|
||||
* Exit codes: 0 success · 1 usage or unrecognized format · 2 refused import
|
||||
* (declared-count mismatch, or a target file that must not be overwritten).
|
||||
*
|
||||
* Known limits:
|
||||
* - Check 2 is check-then-write, not atomic. Two imports running
|
||||
* SIMULTANEOUSLY into one directory can both pass the check before either
|
||||
* writes, and one then clobbers the other. Measured 2026-08-02 over three
|
||||
* independent sets of 40 trials of two concurrent conflicting imports: 19,
|
||||
* 22, and 24 refused out of 40 — roughly half, and it is a race, so expect
|
||||
* the number to move. Against the previous script the same experiment
|
||||
* refused 0 of 40. Closing it needs a lock file, which is a larger change
|
||||
* than this guard. Sequential runs are what this CLI is for, and are what
|
||||
* check 2 covers.
|
||||
* - An id-less conversation cannot be REFRESHED in place. A changed re-import
|
||||
* of an `id: null` export is refused rather than applied, because nothing
|
||||
* in either file distinguishes it from a different conversation at the same
|
||||
* array position. Import it into a fresh directory. This is a deliberate
|
||||
* trade: the same ambiguity, resolved the other way, is what silently
|
||||
* destroyed the earlier import.
|
||||
* - Identity is matched on the conversation id alone, while the filename is
|
||||
* date + id. A conversation whose `created_at` changes between exports
|
||||
* therefore lands on a NEW filename and orphans its earlier page rather
|
||||
* than updating it — duplication, not loss, and true of this script before
|
||||
* these guards existed too.
|
||||
* - THE `updated_at` TIEBREAK IS WITHIN ONE ENVELOPE. It decides which of two
|
||||
* copies in the SAME file survives, and it has no effect across runs: check
|
||||
* 2 treats any existing page carrying this conversation's id as this
|
||||
* export's own page to refresh, so importing an OLDER export after a newer
|
||||
* one replaces the newer page at exit 0 with no warning at all. The
|
||||
* identical stale/fresh pair is decided one way inside an envelope and the
|
||||
* other way across two of them. Both rules are deliberate — the cross-run
|
||||
* one is what makes a re-import able to update its own page — but the
|
||||
* asymmetry is real and the cross-run direction is the silent one.
|
||||
* - A file carrying this importer's own frontmatter shape is treated as this
|
||||
* importer's page. There is no signature, so a hand-written lookalike is
|
||||
* indistinguishable from the real thing.
|
||||
* - A PARSER-LEGIBLE PAGE IS NOT THE SAME AS AN EXTRACTED ONE. parse.ts
|
||||
* accepts a page only when at least 5% of its non-blank lines anchor a turn
|
||||
* (SCORING_MIN_ACCEPTANCE), so a conversation of very long turns still
|
||||
* lands on `no_match` and still extracts nothing. Measured 2026-08-02 on
|
||||
* envelopes built by the reference converter, two turns per side: 25
|
||||
* paragraphs per assistant turn parses (density 0.070, 4 of 4 messages), 40
|
||||
* paragraphs does not (0.046, 0 messages). The exact crossover, measured
|
||||
* line by line: 18 non-blank lines per turn parses (0.0526), 19 does not
|
||||
* (0.0499) — the H1 counts in the denominator too. So turns averaging more
|
||||
* than ~18 non-blank lines of prose fall below the floor. That threshold
|
||||
* lives in gbrain's parser, not here — this script cannot raise it, and
|
||||
* long-form assistant answers sit close to it.
|
||||
* - ★ A PASTED TRANSCRIPT CAN REPLACE THE WHOLE CONVERSATION, and nothing
|
||||
* reports it. This is the sharpest limit here and it is not fixable from
|
||||
* this script.
|
||||
*
|
||||
* A message whose own text contains lines shaped like SOME OTHER export
|
||||
* format — anyone who has pasted a Slack, Discord, Telegram or IRC snippet
|
||||
* into a chat — puts those lines in the body too. parse.ts picks ONE
|
||||
* pattern per page, scored on the first 10 body lines
|
||||
* (SCORING_HEAD_LINES), and only re-scores against the full body when that
|
||||
* head score falls under 0.3. So the pasted block only has to win the head
|
||||
* window; the length of the real conversation is irrelevant. Measured, with
|
||||
* four `**[09:0N] Colleague N:**` lines quoted inside message 1:
|
||||
*
|
||||
* real turns pasted lines winner frontmatter / body turns
|
||||
* 2 2 imessage-slack 2 / 2
|
||||
* 2 3 telegram-bracket 2 / 3
|
||||
* 4 4 telegram-bracket 4 / 4 <- counts AGREE
|
||||
* 40 4 telegram-bracket 40 / 4
|
||||
*
|
||||
* In the last row all forty real turns are gone and four fabricated
|
||||
* speakers at fabricated times reach the fact extractor in their place —
|
||||
* at exit 0, with `phase: regex_match`, so `pages_skipped` stays 0 and
|
||||
* `gbrain doctor` reports `conversation_format_coverage` OK.
|
||||
*
|
||||
* Comparing `frontmatter.messages.length` against the parsed turn count
|
||||
* catches three of those four rows and NOT the 4/4 one, where the counts
|
||||
* agree while every speaker and timestamp is fabricated. Count is a
|
||||
* smoke alarm, not a proof. Closing this needs a change in parse.ts —
|
||||
* fenced-code awareness, or per-pattern scoring that does not let a
|
||||
* ten-line window speak for the page.
|
||||
* - Neither does the parser respect fenced code blocks: a turn header inside
|
||||
* ``` ``` ``` still anchors a turn. `inferTitleFromBody` in markdown.ts
|
||||
* tracks fences; parse.ts does not.
|
||||
* - `messages[i]` is positional. The body carries no id to join on, so an
|
||||
* edit that inserts or removes a turn in the body without editing the
|
||||
* frontmatter silently re-points every id after it — and so does any of the
|
||||
* parser behavior above.
|
||||
* - THE DATE IS THE UTC CALENDAR DAY, NOT THE USER'S. The page `date` and
|
||||
* every turn header are read off a timestamp already normalised to UTC, so
|
||||
* a conversation held in the evening west of Greenwich files on the
|
||||
* FOLLOWING day, and one held in the early morning east of it files on the
|
||||
* PREVIOUS day. Measured here, importing under TZ=America/Los_Angeles a
|
||||
* conversation that happened at 19:30 on Sunday 2 November 2025 in
|
||||
* California:
|
||||
*
|
||||
* created_at "2025-11-03T03:30:00.000Z"
|
||||
* -> date: "2025-11-03" (a Monday)
|
||||
* -> **Me** (2025-11-03 03:30):
|
||||
*
|
||||
* Anything that groups, windows or reports these pages by day inherits that
|
||||
* shift. The importing machine's own zone changes nothing — the output
|
||||
* above is identical under every TZ, deliberately.
|
||||
*
|
||||
* NOT FIXABLE HERE, and the reason is not neglect: the offset is not in the
|
||||
* file this script reads. envelope-v0 renders every timestamp as
|
||||
* `YYYY-MM-DDTHH:mm:ss.sssZ` — "always UTC, always the `Z` designator"
|
||||
* (SPEC.md rule 3) — so an offset a source export DID carry is normalised
|
||||
* away before the envelope reaches this script. Where a source carries no
|
||||
* designator at all, that same rule explains why it is read as UTC rather
|
||||
* than guessed: "The source's true offset is unknowable, and UTC is the
|
||||
* only machine-independent choice."
|
||||
*
|
||||
* Recovering the user's own day would therefore take a FORMAT change — a
|
||||
* conversation-level offset envelope-v0 does not have — and on the evidence
|
||||
* available there would usually be nothing to put in it. Across the
|
||||
* reference converter's own input corpus, 13 conversations carry 26
|
||||
* conversation-level timestamps: 20 are bare unix-epoch numbers (ChatGPT
|
||||
* `create_time`/`update_time`, which cannot express a zone at all), 4 end
|
||||
* in `Z`, 2 carry no designator, and NONE carries a numeric offset. What is
|
||||
* NOT available is guessing from the IMPORTING machine's zone: that would
|
||||
* make one envelope produce different pages on two laptops, which is the
|
||||
* determinism this script is built on.
|
||||
* - MINUTE RESOLUTION IS THE CEILING, and it is the parser's, not this
|
||||
* format's. Every branch of `buildIso` in parse.ts hardcodes `:00` seconds,
|
||||
* and no built-in pattern captures a seconds group — `signal-export`
|
||||
* matches seconds in its regex and still discards them. So two turns in the
|
||||
* same minute come back with identical timestamps: `claude-basic` m1
|
||||
* (15:02:00) and m2 (15:02:31) both parse to `2026-06-14T15:02:00Z`.
|
||||
* Anything downstream that orders or windows on the PARSED timestamp sees a
|
||||
* tie. The frontmatter `ts` keeps full resolution and is the only place on
|
||||
* the page that has it.
|
||||
*
|
||||
* Memory: the whole envelope is held in memory (no streaming); envelopes are
|
||||
* far smaller than the vendor exports they serialize.
|
||||
*
|
||||
* Verify:
|
||||
* node scripts/envelope-to-gbrain.mjs test/fixtures/memvelope/sample.mve.json /tmp/out
|
||||
* -> expect "wrote 1 markdown page(s) (4 message(s))"
|
||||
* bun test test/envelope-to-gbrain.test.ts
|
||||
*
|
||||
* STATUS:
|
||||
* - 2026-07-03, pre-guard behavior, live-verified against gbrain v0.42.56.0:
|
||||
* the sample fixture -> 1 page; a real 662MB Claude export -> 353
|
||||
* conversations = 353 distinct pages (no collisions), searchable after sync
|
||||
* with provenance and message-id citations intact.
|
||||
* - 2026-08-02, the two guards above: verified against all 12 golden fixtures
|
||||
* from the memvelope reference converter and against fresh envelopes
|
||||
* produced by running that converter over synthetic ChatGPT and Claude
|
||||
* exports. All 19 import at exit 0 with zero stderr bytes, and 18 of them
|
||||
* reproduce every message text byte-verbatim. The exception is the
|
||||
* lone-surrogate golden fixture, where an unpaired `U+D800` becomes
|
||||
* `U+FFFD` on UTF-8 write — behavior of `writeFileSync`, unchanged by these
|
||||
* guards and identical on the previous script. Neither guard has been run
|
||||
* against a full-size real export.
|
||||
* - 2026-08-02, the parser-legible format above. Measured on two throwaway
|
||||
* HOME-redirected PGLite brains fed the SAME 13 conversations, one written
|
||||
* the old way and one the new:
|
||||
* `gbrain conversation-parser scan <slug>`, run once per page (it takes
|
||||
* one slug; there is no aggregate form):
|
||||
* 13/13 pages `no_match`, 0 messages
|
||||
* -> 13/13 `imessage-slack`, 33 messages
|
||||
* `gbrain extract-conversation-facts --dry-run`
|
||||
* "Skipped 13 page(s)" (pages_skipped)
|
||||
* -> 0 skipped; every page segments and
|
||||
* reaches the extractor
|
||||
* `gbrain doctor` conversation_format_coverage
|
||||
* warn: "13/13 ... match NO built-in
|
||||
* pattern"
|
||||
* -> ok: "13 pages: imessage-slack=13"
|
||||
* Over all 12 golden fixtures from the reference converter: exit 0, zero
|
||||
* stderr bytes, 33/33 messages parsed, and every `id`/`ts` recovered from
|
||||
* frontmatter byte-identical to the envelope. Every message text is on disk
|
||||
* verbatim except the lone-surrogate fixture noted above; the parser's own
|
||||
* output additionally collapses blank lines WITHIN a message, so a
|
||||
* multi-paragraph turn comes back joined by single newlines.
|
||||
* Not verified: any full-size real export, and any brain with a chat model
|
||||
* configured — the extractor was reached but its LLM call could not run.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const EXIT_REFUSED = 2;
|
||||
|
||||
const [, , envelopePath, outDir = './brain/conversations'] = process.argv;
|
||||
if (!envelopePath) {
|
||||
console.error('usage: node envelope-to-gbrain.mjs <envelope.mve.json> [outDir]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const env = JSON.parse(readFileSync(envelopePath, 'utf8'));
|
||||
if (env.memvelope !== 'envelope-v0') {
|
||||
console.error(`not an envelope-v0 file (memvelope field = ${JSON.stringify(env.memvelope)})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const slug = (s, fallback) =>
|
||||
(String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback).slice(0, 60);
|
||||
|
||||
/** A frontmatter value, emitted as JSON.
|
||||
*
|
||||
* JSON is valid YAML flow syntax, so this is total for any JSON-serializable
|
||||
* value — and, for the thing that matters here, a string always comes out
|
||||
* QUOTED. An unquoted RFC 3339 scalar is read back as a JS `Date`, which is
|
||||
* lossy (microseconds truncated, offset normalised away) and sticky (it
|
||||
* re-serializes unquoted, so it stays a Date on every later round trip). */
|
||||
const yamlJson = (v) => JSON.stringify(v === undefined ? null : v);
|
||||
|
||||
/** The date a turn header is allowed to carry: exactly `YYYY-MM-DD`. */
|
||||
const HEADER_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
/** What `deriveDateContext()` in gbrain's conversation parser falls back to when
|
||||
* a page carries no date at all. Reusing it means a dateless conversation's
|
||||
* headers introduce no value gbrain would not have chosen for itself. */
|
||||
const EPOCH_DATE = '1970-01-01';
|
||||
|
||||
/** The RFC 3339 shapes this script will read a wall clock out of.
|
||||
*
|
||||
* Deliberately NOT `new Date(string)`: for a date-time with no zone
|
||||
* designator, ECMAScript parses local time, so the same envelope would import
|
||||
* differently on two machines and this script claims to be deterministic.
|
||||
* Groups: 1=Y 2=M 3=D 4=hh 5=mm, then an optional offset 6=sign 7=hh 8=mm. */
|
||||
const TS_SHAPE =
|
||||
/^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2})(?::\d{2}(?:\.\d+)?)?(?:[Zz]|([+-])(\d{2}):?(\d{2}))?$/;
|
||||
|
||||
/**
|
||||
* The `YYYY-MM-DD HH:MM` a turn header carries, or null when the message's `ts`
|
||||
* cannot supply one.
|
||||
*
|
||||
* 24-hour, and UTC. `imessage-slack` — the pattern these headers are written
|
||||
* for — declares `timezone_policy: 'inline_utc'`, i.e. gbrain reads the inline
|
||||
* clock AS UTC. So a `+05:30` timestamp must be shifted before it is written;
|
||||
* emitting the local wall clock would record every fact 5.5 hours off. A `Z`
|
||||
* timestamp, or one with no designator at all, is already taken as UTC and its
|
||||
* digits are copied straight across, after the calendar check below — no
|
||||
* arithmetic on the common path, so no hour can be shifted by a conversion.
|
||||
*/
|
||||
function headerClock(ts) {
|
||||
if (typeof ts !== 'string') return null;
|
||||
const m = TS_SHAPE.exec(ts.trim());
|
||||
if (m === null) return null;
|
||||
const [, year, month, day, hour, minute, sign, offsetHour, offsetMinute] = m;
|
||||
const [y, mo, d, h, mi] = [year, month, day, hour, minute].map(Number);
|
||||
// The regex counts digits; it does not know a calendar. Without this it
|
||||
// accepts `2025-99-99T99:99` — and `imessage-slack` MATCHES a header built
|
||||
// from those digits, so gbrain stores an instant no calendar contains.
|
||||
// `2025-02-30` is worse: it yields a VALID Date silently shifted to March 2.
|
||||
// `created_at` is already validated before it reaches a header (see
|
||||
// `pageDate`); the per-message clock is the same untrusted surface and is
|
||||
// used far more often. Numbers only — no string parsing, so no
|
||||
// engine-dependent interpretation of the input; and the UTC setters rather
|
||||
// than `Date.UTC`, which applies MakeFullYear and would read a four-digit
|
||||
// year of `0050` as 1950.
|
||||
if (h > 23 || mi > 59) return null;
|
||||
const utc = new Date(0);
|
||||
utc.setUTCFullYear(y, mo - 1, d);
|
||||
utc.setUTCHours(h, mi, 0, 0);
|
||||
// A date that does not survive its own round trip was never a date: month 99
|
||||
// and February 30 both roll, and the roll is what this catches.
|
||||
if (utc.getUTCFullYear() !== y || utc.getUTCMonth() !== mo - 1 || utc.getUTCDate() !== d) {
|
||||
return null;
|
||||
}
|
||||
// No offset: the digits are already UTC by this script's policy, so they are
|
||||
// copied across rather than reformatted. This is the common path, and it does
|
||||
// no arithmetic at all.
|
||||
if (sign === undefined) return `${year}-${month}-${day} ${hour}:${minute}`;
|
||||
const [oh, om] = [offsetHour, offsetMinute].map(Number);
|
||||
if (oh > 23 || om > 59) return null;
|
||||
utc.setUTCMinutes(utc.getUTCMinutes() - (oh * 60 + om) * (sign === '-' ? -1 : 1));
|
||||
const pad = (n, width = 2) => String(n).padStart(width, '0');
|
||||
// The year is padded to four digits like every other field: the pattern's
|
||||
// regex requires `\d{4}`, so an unpadded `49` would emit a header that does
|
||||
// not parse at all — a turn silently merged into its neighbour.
|
||||
return `${pad(utc.getUTCFullYear(), 4)}-${pad(utc.getUTCMonth() + 1)}-${pad(utc.getUTCDate())} ${pad(utc.getUTCHours())}:${pad(utc.getUTCMinutes())}`;
|
||||
}
|
||||
|
||||
/** The RFC 3339 shapes a CONVERSATION-level `updated_at` is ordered by.
|
||||
*
|
||||
* Deliberately a second regex rather than `TS_SHAPE`: that one exists to build
|
||||
* a turn header, whose resolution is the minute, so it discards seconds. Two
|
||||
* exports of one conversation are routinely closer together than that, and the
|
||||
* reference converter emits milliseconds always (SPEC.md rule 3 renders every
|
||||
* timestamp as `YYYY-MM-DDTHH:mm:ss.sssZ`), so seconds and fraction are
|
||||
* captured here.
|
||||
* Groups: 1=Y 2=M 3=D 4=hh 5=mm 6=ss 7=.fff, then an offset 8=sign 9=hh 10=mm. */
|
||||
const UPDATED_AT_SHAPE =
|
||||
/^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2})(?::(\d{2})(\.\d+)?)?(?:[Zz]|([+-])(\d{2}):?(\d{2}))?$/;
|
||||
|
||||
/**
|
||||
* The instant `updated_at` names, in epoch milliseconds, or null when the value
|
||||
* is not one this script will order by.
|
||||
*
|
||||
* A NUMBER, not a string comparison: `2026-06-09T02:00+05:30` sorts above
|
||||
* `2026-06-08T23:00Z` lexically and is two and a half hours EARLIER as an
|
||||
* instant. And not `new Date(string)`: a date-time with no zone designator is
|
||||
* parsed as LOCAL time by ECMAScript, so the same pair of envelopes would
|
||||
* resolve differently on two machines, which this script promises not to do. No
|
||||
* designator means UTC here, matching both `headerClock` and the spec, whose
|
||||
* reasoning is that the source's true offset is unknowable.
|
||||
*
|
||||
* Same calendar discipline as `headerClock`: the regex counts digits, so
|
||||
* `2026-02-30` reaches it as a well-formed string that `Date` silently rolls to
|
||||
* March 2. A value that does not survive its own round trip is not a date, and
|
||||
* an envelope is third-party input.
|
||||
*/
|
||||
function updatedAtInstant(value) {
|
||||
if (typeof value !== 'string') return null;
|
||||
const m = UPDATED_AT_SHAPE.exec(value.trim());
|
||||
if (m === null) return null;
|
||||
const [, year, month, day, hour, minute, second, fraction, sign, offsetHour, offsetMinute] = m;
|
||||
const [y, mo, d, h, mi] = [year, month, day, hour, minute].map(Number);
|
||||
if (h > 23 || mi > 59) return null;
|
||||
// 60 is a leap second, which RFC 3339 permits and which names a real instant.
|
||||
// It is added AFTER the calendar check below, since `23:59:60` legitimately
|
||||
// rolls the date and that roll must not be read as an impossible date.
|
||||
const s = second === undefined ? 0 : Number(second);
|
||||
if (s > 60) return null;
|
||||
const utc = new Date(0);
|
||||
utc.setUTCFullYear(y, mo - 1, d);
|
||||
utc.setUTCHours(h, mi, 0, 0);
|
||||
if (utc.getUTCFullYear() !== y || utc.getUTCMonth() !== mo - 1 || utc.getUTCDate() !== d) {
|
||||
return null;
|
||||
}
|
||||
// Kept as a number rather than pushed back through `Date`, so a
|
||||
// sub-millisecond fraction still participates in the comparison — down to
|
||||
// whatever a double has left at epoch scale, which is roughly a microsecond
|
||||
// in this century. The reference producer emits exactly three fractional
|
||||
// digits (SPEC.md rule 3), so nothing it can write reaches that floor.
|
||||
let ms = utc.getTime() + s * 1000 + (fraction === undefined ? 0 : Number(fraction) * 1000);
|
||||
if (sign !== undefined) {
|
||||
const [oh, om] = [offsetHour, offsetMinute].map(Number);
|
||||
if (oh > 23 || om > 59) return null;
|
||||
ms -= (oh * 60 + om) * 60000 * (sign === '-' ? -1 : 1);
|
||||
}
|
||||
return ms;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which of two conversations sharing one target filename is kept.
|
||||
*
|
||||
* `later` is the one further along `conversations[]`; before this existed it
|
||||
* simply won, and that is the defect. A merged re-export is the mainstream
|
||||
* path — the memvelope CLI's own USAGE tells users to pass every downloaded
|
||||
* file at once, the spec forbids the converter from re-sorting them, and folder
|
||||
* expansion sorts by FILENAME. Every automatic duplicate-namer a browser or OS
|
||||
* applies to a second download of `conversations.json` inserts a character that
|
||||
* sorts below `.` (` (1)`, `(1)`, `-1`, ` 2`), so the RE-EXPORT sorts first and
|
||||
* the ORIGINAL sorts last. Array order was therefore not arbitrary: it was
|
||||
* deterministically wrong, and it kept the stale copy every time.
|
||||
*
|
||||
* `updated_at` is what decides instead. It is a required conversation key in
|
||||
* envelope-v0, both vendor paths of the reference converter populate it
|
||||
* (ChatGPT `update_time`, Claude `updated_at`), and nothing here read it.
|
||||
*
|
||||
* THE FALLBACK, and why it is array order rather than a cleverer guess:
|
||||
*
|
||||
* - Equal instants. Nothing distinguishes the two copies, so the rule that
|
||||
* was there before decides. Changing it would only trade one arbitrary
|
||||
* answer for another, and this one is already pinned by test.
|
||||
* - Comparable on only ONE side — absent (the spec allows `null` with no
|
||||
* fallback), non-string, or a string this script will not order by. Both
|
||||
* copies came out of ONE converter run, so the asymmetry is the vendor's:
|
||||
* one source export carried the field for this conversation and the other
|
||||
* did not. Nothing in that fact says which export is newer — a vendor may
|
||||
* have started emitting the field or stopped — so preferring the copy that
|
||||
* HAS a timestamp is a guess dressed as a rule. The tiebreak is applied
|
||||
* only when BOTH copies carry an orderable `updated_at`.
|
||||
*
|
||||
* BE PLAIN ABOUT WHAT THAT COSTS. On that slice this rule changes nothing:
|
||||
* array order decides, and array order is the same deterministically-wrong
|
||||
* answer described above, so the stale copy still wins. `updated_at: null`
|
||||
* is producer-reachable — `converter.js` ends both normalizers'
|
||||
* `updated_at` with `|| null` — though it occurs in 0 of the 13
|
||||
* conversations in the reference corpus. What this rule buys on that slice
|
||||
* is only that the outcome is LOUD: stderr prints both values and says
|
||||
* array order decided.
|
||||
* - `created_at` is deliberately not a secondary key. It is when the
|
||||
* conversation began, which is identical in both copies of a re-export and
|
||||
* says nothing about which export is newer.
|
||||
* - THE REDUCTION IS PAIRWISE, folded over `conversations[]` in order. With
|
||||
* every copy orderable that is a true maximum. With three or more copies
|
||||
* where one is NOT orderable, the fold loses transitivity and the freshest
|
||||
* copy overall can still be discarded: `[later, absent, earlier]` keeps
|
||||
* `earlier`, because neither comparison had evidence on both sides. That is
|
||||
* the fallback above doing exactly what it says rather than a separate
|
||||
* defect, and it is what the previous script did too.
|
||||
*
|
||||
* Either way a collision is a collision: one copy is discarded, and stderr says
|
||||
* which, why, and with what values.
|
||||
*/
|
||||
function keepsLaterInArray(earlier, later) {
|
||||
const a = updatedAtInstant(earlier);
|
||||
const b = updatedAtInstant(later);
|
||||
if (a === null || b === null || a === b) return { keepLater: true, byUpdatedAt: false };
|
||||
return { keepLater: b > a, byUpdatedAt: true };
|
||||
}
|
||||
|
||||
/** The file's contents, or null if it does not exist. Any other error is the
|
||||
* caller's problem to fail on — an unreadable target must never be silently
|
||||
* treated as an absent one, because "absent" is the answer that permits a
|
||||
* write. */
|
||||
function readIfPresent(path) {
|
||||
try {
|
||||
return readFileSync(path, 'utf8');
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** The conversation identity recorded in a page this importer previously wrote,
|
||||
* or null if the file is not recognizably one of ours.
|
||||
*
|
||||
* A deliberate line scan rather than a YAML parse: this script has no
|
||||
* dependencies, and anything it cannot confidently recognize must fall
|
||||
* through to "foreign" — the answer that refuses the overwrite. `{ id: null }`
|
||||
* means "ours, but written from a conversation that carried no id", which is
|
||||
* a different thing from "not ours" and must not be collapsed into it.
|
||||
*
|
||||
* The id scalar is accepted in every shape a YAML round trip produces, not
|
||||
* only the JSON this importer writes. gbrain rewrites pages it holds —
|
||||
* `export --dir`, the DB-only restore path, and put_page write-through all
|
||||
* re-emit frontmatter through gray-matter, which writes a UUID as a plain
|
||||
* unquoted scalar and an all-digit or boolean-looking id single-quoted.
|
||||
* Recognizing only our own JSON meant every one of those rewrites turned the
|
||||
* page "foreign" and a later refresh refused the whole envelope, advising
|
||||
* the user to delete gbrain's own copy. */
|
||||
function idScalar(rawValue) {
|
||||
if (rawValue.startsWith('"')) {
|
||||
// Our own emitted shape (JSON is valid YAML flow syntax).
|
||||
try {
|
||||
const value = JSON.parse(rawValue);
|
||||
return typeof value === 'string' ? value : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (rawValue.startsWith("'")) {
|
||||
// YAML single-quoted: the only escape is a doubled quote.
|
||||
if (rawValue.length < 2 || !rawValue.endsWith("'")) return null;
|
||||
const body = rawValue.slice(1, -1).replace(/''/g, '\u0000');
|
||||
if (body.includes("'")) return null;
|
||||
const value = body.replace(/\u0000/g, "'");
|
||||
// An empty id is not a shape this importer ever writes; stay foreign,
|
||||
// exactly as the JSON-only reader did.
|
||||
return value === '' ? null : value;
|
||||
}
|
||||
// Plain scalar. `null`/`~`/empty are YAML null, not a string id — and this
|
||||
// importer never writes the key for a null id, so that shape stays foreign.
|
||||
if (rawValue === '' || rawValue === 'null' || rawValue === '~') return null;
|
||||
return rawValue;
|
||||
}
|
||||
|
||||
function existingPageIdentity(raw) {
|
||||
// A page we wrote can pick up cosmetic byte changes without ceasing to be
|
||||
// ours: a git checkout with core.autocrlf, a cross-platform sync, an editor
|
||||
// that adds a BOM. Refusing to recognize those made a whole envelope
|
||||
// unimportable over a line ending, so normalize them away before the scan.
|
||||
const text = raw.replace(/^/, '').replace(/\r\n/g, '\n');
|
||||
if (!text.startsWith('---\n')) return null;
|
||||
const end = text.indexOf('\n---\n', 3);
|
||||
if (end === -1) return null;
|
||||
const ID_KEY = 'memvelope_conversation_id: ';
|
||||
let ours = false;
|
||||
let id = null;
|
||||
for (const line of text.slice(4, end).split('\n')) {
|
||||
if (line === 'origin: memvelope/envelope-v0') {
|
||||
ours = true;
|
||||
} else if (line.startsWith(ID_KEY)) {
|
||||
const value = idScalar(line.slice(ID_KEY.length));
|
||||
if (value === null) return null;
|
||||
id = value;
|
||||
}
|
||||
}
|
||||
return ours ? { id } : null;
|
||||
}
|
||||
|
||||
const conversations = env.conversations || [];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Check 1 — the envelope's own declared counts, before anything is written.
|
||||
//
|
||||
// Each count is judged on its own. Treating "either field exists" as "the
|
||||
// envelope is checkable" gave a half-declared envelope a half check and total
|
||||
// silence, which is the very defect this guard exists to close.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** How a declared count is to be read: a usable number, absent, or present but
|
||||
* not a count at all. The third case must not collapse into the second —
|
||||
* saying "declares no count" about a file that declares a broken one is a
|
||||
* false statement, and it would be printed over a real truncation. */
|
||||
function readDeclaredCount(value) {
|
||||
if (value === undefined) return { state: 'absent' };
|
||||
if (Number.isInteger(value) && value >= 0) return { state: 'declared', value };
|
||||
return { state: 'malformed' };
|
||||
}
|
||||
|
||||
const actualConversations = conversations.length;
|
||||
const actualMessages = conversations.reduce((sum, c) => sum + (c.messages || []).length, 0);
|
||||
const counts = [
|
||||
{ field: 'meta.conversation_count', raw: env.meta?.conversation_count, actual: actualConversations },
|
||||
{ field: 'meta.message_count', raw: env.meta?.message_count, actual: actualMessages },
|
||||
].map((c) => ({ ...c, ...readDeclaredCount(c.raw) }));
|
||||
|
||||
const malformed = counts.filter((c) => c.state === 'malformed');
|
||||
if (malformed.length) {
|
||||
// envelope-v0 types both counts as non-negative integers. A count that is
|
||||
// present but is not one cannot be compared, and an envelope this malformed
|
||||
// is not a file to trust with an unchecked import.
|
||||
console.error('refusing to import: the envelope declares a count that is not a non-negative integer.');
|
||||
for (const c of malformed) console.error(` ${c.field} = ${JSON.stringify(c.raw)}`);
|
||||
console.error('Nothing was written. Re-export, or correct the declared counts if the contents are known-good.');
|
||||
process.exit(EXIT_REFUSED);
|
||||
}
|
||||
|
||||
const mismatched = counts.filter((c) => c.state === 'declared' && c.value !== c.actual);
|
||||
if (mismatched.length) {
|
||||
// Fail closed. The counts are the envelope's own statement of what it holds,
|
||||
// and they disagree with what it holds — so the file is not what it claims,
|
||||
// and nothing here can tell which conversations or turns went missing. A
|
||||
// partial import that exits 0 is how an archive silently becomes a fragment.
|
||||
console.error("refusing to import: the envelope's declared counts disagree with its contents.");
|
||||
// Print both counts, not only the failing one: seeing which half agrees is
|
||||
// what tells a truncated download apart from a broken converter.
|
||||
for (const c of counts) {
|
||||
const declared = c.state === 'declared' ? c.value : 'not declared';
|
||||
console.error(` ${c.field} declared ${declared}, envelope contains ${c.actual}`);
|
||||
}
|
||||
console.error('This envelope is truncated, hand-edited, or from a broken converter. Nothing was written. Re-export, or correct the declared counts if the contents are known-good.');
|
||||
process.exit(EXIT_REFUSED);
|
||||
}
|
||||
|
||||
const absent = counts.filter((c) => c.state === 'absent');
|
||||
if (absent.length) {
|
||||
// envelope-v0 requires both fields, so this file is already non-conforming.
|
||||
// Import it anyway — hand-authored envelopes are useful — but never let an
|
||||
// unchecked import look identical to a checked one on the way past. Naming
|
||||
// the missing field matters: with one count present, only half the envelope
|
||||
// was verified, and the receipt alone cannot show which half.
|
||||
console.warn(
|
||||
`warning: envelope declares no ${absent.map((c) => c.field).join(' and no ')} (envelope-v0 requires both) — integrity check skipped for ${absent.length === 2 ? 'conversations and messages' : absent[0].field.replace('meta.', '').replace('_count', 's')}; a truncated envelope would import silently.`,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Render every page in memory first. Rendering has no side effects, so the
|
||||
// conflict check below can see the complete set of target files — including the
|
||||
// final content of any filename an envelope writes more than once — while the
|
||||
// output directory is still untouched.
|
||||
// ---------------------------------------------------------------------------
|
||||
const pages = new Map();
|
||||
let collisions = 0;
|
||||
for (const [i, c] of conversations.entries()) {
|
||||
const date = (c.created_at || '').slice(0, 10);
|
||||
// Name the file by the conversation's own id, so two conversations that share
|
||||
// a date and title can never silently overwrite each other. The KEY IS THE
|
||||
// PAIR: date and id together name the file, and the date is not merely a
|
||||
// human/chronological prefix — a conversation whose `created_at` changes
|
||||
// between exports lands on a new filename, which is the "orphans its earlier
|
||||
// page" limit in the header. Positional fallback keeps names unique and
|
||||
// deterministic when an envelope omits an id.
|
||||
// One predicate for "this conversation carries its own id", shared by the
|
||||
// filename, the frontmatter below, and the conflict check further down.
|
||||
// Keeping it in a single place is what stops them disagreeing about whether
|
||||
// an id exists.
|
||||
const hasId = typeof c.id === 'string' && c.id.trim() !== '';
|
||||
const convId = hasId ? c.id.trim() : `conv-${i + 1}`;
|
||||
// `date` is third-party, exactly like `convId`, so it gets the same slug()
|
||||
// treatment. Interpolating it raw let a `created_at` of `../…` resolve the
|
||||
// join below outside outDir and write there.
|
||||
const name = `${slug(date, '0000-00-00')}-${slug(convId, `conv-${i + 1}`)}.md`;
|
||||
const messages = c.messages || [];
|
||||
// The date a turn header falls back to when its own message carries no usable
|
||||
// `ts`. `date` is third-party and only length-limited, so it is validated
|
||||
// rather than trusted: a `created_at` of "1\nowner: z" slices to ten
|
||||
// characters that include a newline, and interpolating that into a header
|
||||
// would break the turn it is supposed to anchor.
|
||||
const pageDate = HEADER_DATE.test(date) ? date : EPOCH_DATE;
|
||||
// ONE fallback for an absent title, shared by the frontmatter and the H1.
|
||||
// They used to disagree — "Untitled conversation" above, "Conversation" in
|
||||
// the body — which is two different names for the same missing thing, and
|
||||
// `parseMarkdown` prefers the body's H1 when frontmatter has no title.
|
||||
const title = c.title || 'Untitled conversation';
|
||||
// gbrain reads YAML frontmatter + markdown body; keep provenance in frontmatter.
|
||||
// Emit `type: conversation` so gbrain stores these as conversation pages rather
|
||||
// than defaulting to the generic `concept`. gbrain is open-typed — it takes an
|
||||
// explicit frontmatter `type` verbatim — and its conversation-aware features
|
||||
// (conversation-facts extraction, the conversation_format_coverage check,
|
||||
// chronicle eligibility) key off `type == 'conversation'`.
|
||||
const front = [
|
||||
'---',
|
||||
'type: conversation',
|
||||
// Every interpolated value below is quoted. An envelope is a third-party
|
||||
// file, so any string carrying a newline would otherwise close its scalar
|
||||
// and inject arbitrary frontmatter keys into the page gbrain ingests — or
|
||||
// duplicate an existing key, which makes the parse throw and silently
|
||||
// strips every provenance field from the page.
|
||||
`title: ${JSON.stringify(title)}`,
|
||||
// `date` is the first 10 chars of the envelope's `created_at`; 10 is plenty
|
||||
// to smuggle a newline plus a short key. Absent stays an unquoted YAML null.
|
||||
`date: ${date ? JSON.stringify(date) : 'null'}`,
|
||||
`source: ${JSON.stringify(env.meta?.source_provider || 'unknown')}`,
|
||||
// Omit the key entirely when the envelope carries no id, rather than
|
||||
// emitting the literal `undefined` or a synthesized `conv-N` — the positional
|
||||
// fallback names the file, but it is not a memvelope conversation id and
|
||||
// must not be recorded as one.
|
||||
// The id VERBATIM, not the trimmed form used for the filename. The spec has
|
||||
// converters copy ids exactly, and recording the trimmed one made two ids
|
||||
// differing only by surrounding whitespace indistinguishable on disk — so
|
||||
// the conflict check below read them as one conversation and let the second
|
||||
// import destroy the first.
|
||||
...(hasId ? [`memvelope_conversation_id: ${JSON.stringify(c.id)}`] : []),
|
||||
'origin: memvelope/envelope-v0',
|
||||
// Per-message identity. It cannot ride in the turn header: the only header
|
||||
// shape gbrain's parser reads carries `YYYY-MM-DD HH:MM` and nothing else,
|
||||
// so a message id and a full RFC 3339 timestamp have to live here or be
|
||||
// thrown away. Order is the body's order, so `messages[i]` is the i-th turn.
|
||||
//
|
||||
// An array of maps, deliberately. Not a map keyed by id: order IS the
|
||||
// join — `messages[i]` is body turn i — and a mapping discards it. (The
|
||||
// duplicate-id argument belongs to CONVERSATION ids, which the spec says
|
||||
// consumers must tolerate; message ids are positional per spec and unique
|
||||
// within their conversation.) And not one packed string per message, which
|
||||
// asks a consumer to split on a space and breaks the moment an id has one.
|
||||
//
|
||||
// An explicit `[]` rather than an omitted key: omission is
|
||||
// indistinguishable from a page written before this format existed, and a
|
||||
// consumer reading identity back needs to tell those apart.
|
||||
...(messages.length === 0
|
||||
? ['messages: []']
|
||||
: [
|
||||
'messages:',
|
||||
...messages.flatMap((m) => [` - id: ${yamlJson(m.id)}`, ` ts: ${yamlJson(m.ts)}`]),
|
||||
]),
|
||||
'---',
|
||||
'',
|
||||
].join('\n');
|
||||
const body = messages
|
||||
.map((m) => {
|
||||
// The message's OWN date, not the conversation's: `imessage-slack` is an
|
||||
// inline-date pattern precisely so a conversation spanning midnight lands
|
||||
// its turns on the right days.
|
||||
const clock = headerClock(m.ts) || `${pageDate} 00:00`;
|
||||
return `**${m.role === 'user' ? 'Me' : 'Assistant'}** (${clock}):\n\n${m.text}`;
|
||||
})
|
||||
// No `---` rule between turns. A horizontal rule is a non-blank line that
|
||||
// matches no pattern, so the parser appends it to the preceding message:
|
||||
// every extracted message text ended `...\n---`. It also diluted the
|
||||
// match-density score the parser's acceptance floor is computed from.
|
||||
.join('\n\n');
|
||||
const rendered = {
|
||||
// The H1 is the ONLY place a third-party string reaches the body, and the
|
||||
// body is now parsed. A title carrying a newline used to look merely
|
||||
// untidy; since the turn headers became legible it manufactures a TURN, and
|
||||
// one that lands ahead of every real one — so `messages[0]` in frontmatter
|
||||
// names content the user never sent and every id after it is off by one.
|
||||
// The heading is flattened to a single line for that reason; the verbatim
|
||||
// title, newlines and all, is still recorded in the frontmatter above.
|
||||
content: front + `# ${title.replace(/\s*[\r\n]+\s*/g, ' ')}\n\n` + body + '\n',
|
||||
messageCount: messages.length,
|
||||
// The conversation's OWN id, verbatim, or null. Never the positional
|
||||
// fallback: that is a filename, not an identity, and treating it as one is
|
||||
// the whole bug. Verbatim rather than trimmed for the same reason — see the
|
||||
// frontmatter note above.
|
||||
conversationId: hasId ? c.id : null,
|
||||
// Carried only to break a filename collision. It is NOT written to the
|
||||
// page — no frontmatter key holds it, by design and separately tracked.
|
||||
updatedAt: c.updated_at,
|
||||
};
|
||||
const earlier = pages.get(name);
|
||||
if (earlier === undefined) {
|
||||
pages.set(name, rendered);
|
||||
continue;
|
||||
}
|
||||
// Never lose a page silently: two conversations mapping to the same filename
|
||||
// (an envelope carrying duplicate ids — which the spec permits, since merging
|
||||
// never deduplicates — or DISTINCT ids that slug alike, which the header
|
||||
// describes and which this warning's wording does not cover) means one of
|
||||
// them is discarded. Warn loudly rather than overwrite in silence, and report
|
||||
// the count of DISTINCT files written — not the number of write calls, which
|
||||
// is what hid the old title-collision bug.
|
||||
collisions += 1;
|
||||
const { keepLater, byUpdatedAt } = keepsLaterInArray(earlier.updatedAt, rendered.updatedAt);
|
||||
// Name the decision AND its inputs. "Overwriting the earlier page" was the
|
||||
// whole message before, and it would now be false half the time — the reader
|
||||
// has to be able to check which copy survived rather than assume the old
|
||||
// rule still applies.
|
||||
const verdict = byUpdatedAt
|
||||
? `keeping the copy whose updated_at is later (${JSON.stringify(keepLater ? rendered.updatedAt : earlier.updatedAt)}) over ${JSON.stringify(keepLater ? earlier.updatedAt : rendered.updatedAt)}`
|
||||
: `updated_at cannot order these two copies (${JSON.stringify(earlier.updatedAt ?? null)} and ${JSON.stringify(rendered.updatedAt ?? null)}), so array order decides — overwriting the earlier page`;
|
||||
console.warn(`warning: filename collision on "${name}" — conversation id ${JSON.stringify(c.id)} is not unique; ${verdict}.`);
|
||||
if (keepLater) pages.set(name, rendered);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Check 2 — target files that already exist and were not written by this run.
|
||||
// `pages` is per-process and the default outDir is a fixed literal, so without
|
||||
// this a second import into the same directory clobbered the first in silence.
|
||||
// Only the exact target filenames are examined: unrelated markdown sitting in
|
||||
// the output directory is none of this script's business.
|
||||
// ---------------------------------------------------------------------------
|
||||
const conflicts = [];
|
||||
for (const [name, page] of pages) {
|
||||
const existing = readIfPresent(join(outDir, name));
|
||||
// Absent, or already exactly what we are about to write (a re-import of the
|
||||
// same envelope). Rewriting identical bytes changes nothing.
|
||||
if (existing === null || existing === page.content) continue;
|
||||
const identity = existingPageIdentity(existing);
|
||||
if (identity === null) {
|
||||
// Say what is true — the file was not recognized. Asserting that this
|
||||
// importer did not write it is a claim this code is in no position to make,
|
||||
// and it is wrong for any page of ours that has been edited since.
|
||||
conflicts.push(` ${name} — already exists and could not be recognized as a page written by this importer.`);
|
||||
} else if (identity.id === null) {
|
||||
// Ours, but written from an id-less conversation, so its filename encodes
|
||||
// array position rather than identity. An update and a wholly different
|
||||
// conversation are indistinguishable here; guessing either way risks
|
||||
// destroying an import.
|
||||
conflicts.push(` ${name} — written by this importer from a conversation with no id, so it cannot be matched to this envelope's conversation. Refusing to guess.`);
|
||||
} else if (identity.id !== page.conversationId) {
|
||||
// Distinct ids that slug to one filename (truncation at 60 chars, or
|
||||
// characters that slug away). Rare, but silently fatal if permitted.
|
||||
conflicts.push(` ${name} — holds conversation ${JSON.stringify(identity.id)}, but this envelope maps ${JSON.stringify(page.conversationId)} to the same filename.`);
|
||||
}
|
||||
// Otherwise: same conversation id, different content — a refreshed export
|
||||
// updating its own page. That is exactly what re-importing is for.
|
||||
}
|
||||
|
||||
if (conflicts.length) {
|
||||
console.error(`refusing to import: ${conflicts.length} target file(s) in ${outDir} would be overwritten with different content.`);
|
||||
for (const line of conflicts) console.error(line);
|
||||
console.error('Nothing was written. Import into a different output directory, or delete the listed file(s) if they are stale.');
|
||||
process.exit(EXIT_REFUSED);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Write. Everything above has already passed, so this loop cannot refuse.
|
||||
// ---------------------------------------------------------------------------
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
let messagesWritten = 0;
|
||||
for (const [name, page] of pages) {
|
||||
writeFileSync(join(outDir, name), page.content);
|
||||
messagesWritten += page.messageCount;
|
||||
}
|
||||
|
||||
console.log(`wrote ${pages.size} markdown page(s) (${messagesWritten} message(s)) to ${outDir} — point gbrain's sync at this directory.`);
|
||||
if (collisions) {
|
||||
// "Overwritten" would now be false whenever the tiebreak kept the earlier
|
||||
// copy: that copy is never rewritten and the later one is never written at
|
||||
// all. "Discarded" is true in both directions, and the per-collision lines
|
||||
// above already say which copy went.
|
||||
console.warn(`warning: ${collisions} filename collision(s) — ${collisions} page(s) discarded. Deduplicate conversation ids in the envelope to avoid data loss.`);
|
||||
}
|
||||
if (messagesWritten !== actualMessages) {
|
||||
// The page count alone cannot show this: a discarded copy leaves the same one
|
||||
// file on disk, so only the message tally reveals the turns that went with it.
|
||||
//
|
||||
// "Discarded", not "overwritten", for the same reason as the summary line
|
||||
// above: since the tiebreak can keep the EARLIER copy, the losing copy is
|
||||
// sometimes never written at any point.
|
||||
//
|
||||
// Worded as a fact about the discarded copies, not as an announcement of
|
||||
// loss. Duplicate ids are conforming input — the spec has merging never
|
||||
// deduplicate — so converting an old export together with a newer one, which
|
||||
// is what the memvelope CLI tells users to do, lands here routinely with the
|
||||
// surviving page already holding every unique turn. An alarm that cries wolf
|
||||
// on the mainstream path teaches its reader to ignore the one that matters.
|
||||
console.warn(`warning: the discarded page(s) carried ${actualMessages - messagesWritten} message(s) that are not on disk (${actualMessages} read, ${messagesWritten} written). If they were earlier copies of the same conversation, the surviving page may already contain those turns; if not, this is real loss.`);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Regenerates skills/skills.lock.json — the tamper-evidence manifest mapping
|
||||
* every bundled file under skills/ to its sha256 (#159). Not a signature
|
||||
* system: it turns silent skill edits into explicit diffs. `gbrain doctor`
|
||||
* warns (never fails) on drift; scripts/check-skills-manifest-fresh.sh keeps
|
||||
* the committed manifest in sync in CI.
|
||||
*
|
||||
* Run after any change under skills/:
|
||||
* bun run scripts/generate-skills-manifest.ts
|
||||
*/
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
SKILLS_MANIFEST_FILENAME,
|
||||
renderSkillsManifest,
|
||||
} from '../src/core/skills-integrity.ts';
|
||||
|
||||
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const skillsDir = join(repoRoot, 'skills');
|
||||
const outPath = join(skillsDir, SKILLS_MANIFEST_FILENAME);
|
||||
writeFileSync(outPath, renderSkillsManifest(skillsDir));
|
||||
console.log(`Wrote ${outPath}`);
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bun
|
||||
// scripts/postinstall.ts
|
||||
//
|
||||
// Postinstall hook: after `bun install`, apply any pending schema migrations so
|
||||
// a freshly-installed gbrain is immediately usable. Wired via package.json
|
||||
// ("postinstall": "bun run scripts/postinstall.ts") as a real Bun script rather
|
||||
// than an inline `node -e` one-liner.
|
||||
//
|
||||
// Why a script file and not an inline command:
|
||||
// Embedding a program inside the package.json postinstall string lets the
|
||||
// lifecycle shell mangle it. Bun's Windows script-runner expands `\n` in the
|
||||
// hint string into a REAL newline before node sees it, producing
|
||||
// `SyntaxError: Invalid or unexpected token` and aborting the whole install.
|
||||
// `node` is also not guaranteed present under a Bun install (bun is the
|
||||
// guaranteed runtime), and `shell: win32` re-opens a quoting surface. A
|
||||
// checked-in .ts run by `bun run` sidesteps all three.
|
||||
//
|
||||
// Uses Bun APIs only — `which()` for Windows-aware PATH resolution (finds
|
||||
// gbrain.exe / gbrain.cmd) and an argv-array `Bun.spawnSync` (no shell, nothing
|
||||
// to quote). It NEVER fails the install: every path exits 0.
|
||||
|
||||
import { which } from 'bun';
|
||||
|
||||
const HINT =
|
||||
'[gbrain] postinstall skipped. If installed via bun install -g github:...: ' +
|
||||
'run `gbrain doctor` and `gbrain apply-migrations --yes` manually. ' +
|
||||
'See https://github.com/garrytan/gbrain/issues/218';
|
||||
|
||||
// Windows-aware PATH resolution — finds gbrain, gbrain.exe or gbrain.cmd.
|
||||
const bin = which('gbrain');
|
||||
|
||||
if (!bin) {
|
||||
// Fresh clone / global install where gbrain isn't on PATH yet: skip cleanly.
|
||||
console.error(HINT);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
const r = Bun.spawnSync({
|
||||
cmd: [bin, 'apply-migrations', '--yes', '--non-interactive'],
|
||||
stdout: 'inherit',
|
||||
stderr: 'inherit',
|
||||
});
|
||||
if (r.exitCode !== 0) console.error(HINT);
|
||||
} catch {
|
||||
console.error(HINT);
|
||||
}
|
||||
|
||||
process.exit(0); // never abort the install
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user