mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 17:02:19 +00:00
Compare commits
3
Commits
v0.45.11.0
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d941e9f918 | ||
|
|
26578a2478 | ||
|
|
7fdcd8bd2e |
@@ -1,7 +1,7 @@
|
||||
name: Heavy Tests
|
||||
|
||||
# Heavy ops-shape tests under tests/heavy/. Cost minutes per run; NOT part
|
||||
# of default PR CI. Two triggers:
|
||||
# of default PR CI. Three triggers:
|
||||
# - Nightly schedule (catches regressions within 24h of merge to master).
|
||||
# - On-demand opt-in via PR label `heavy-tests` (slow loop kept off by default).
|
||||
# - Manual workflow_dispatch for triage.
|
||||
@@ -102,14 +102,15 @@ jobs:
|
||||
retention-days: 14
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Real-agent door e2e: drives the ACTUAL `claude` + `codex` binaries (no PATH
|
||||
# shims) against a real gbrain over MCP. These pay real API cost and need the
|
||||
# binaries installed + authed, which a stock GitHub runner does NOT have — so
|
||||
# both tests self-SKIP (describe.skipIf on binary/auth) and the job is a clean
|
||||
# no-op here. It exists so a self-hosted / manually-provisioned runner WITH
|
||||
# authed claude/codex (and ANTHROPIC/OPENAI creds) actually exercises the real
|
||||
# binaries. Heavy cadence only (nightly + `real-agent-e2e` label + dispatch);
|
||||
# NEVER the PR shard matrix.
|
||||
# Real-agent door e2e: drives the ACTUAL `claude` + `codex` + `hermes`
|
||||
# binaries (no PATH shims) against a real gbrain over MCP. These pay real API
|
||||
# cost and need the binaries installed + authed, which a stock GitHub runner
|
||||
# does NOT have — so the tests self-SKIP (describe.skipIf on binary/auth) and
|
||||
# the job is a clean no-op here. It exists so a self-hosted /
|
||||
# manually-provisioned runner WITH authed claude/codex/hermes (and
|
||||
# ANTHROPIC/OPENAI creds) actually exercises the real binaries. Heavy cadence
|
||||
# only (nightly + `real-agent-e2e` label + dispatch); NEVER the PR shard
|
||||
# matrix.
|
||||
real-agent-e2e:
|
||||
name: Real-agent door e2e (skips without authed binaries)
|
||||
if: |
|
||||
@@ -118,6 +119,10 @@ jobs:
|
||||
contains(github.event.pull_request.labels.*.name, 'heavy-tests')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# Open the hermes opt-in door here so binary/auth absence — not the
|
||||
# opt-in var — is what skips (same posture as the claude/codex doors).
|
||||
GBRAIN_REAL_HERMES_E2E: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
@@ -125,15 +130,16 @@ jobs:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
|
||||
# Reference both door tests; run only the ones present (the claude door
|
||||
# may land in a sibling PR). Missing binary/auth → the file self-skips, so
|
||||
# a stock runner reports a green no-op rather than failing.
|
||||
# Reference the door tests; run only the ones present (a door may land
|
||||
# in a sibling PR). Missing binary/auth → the file self-skips, so a
|
||||
# stock runner reports a green no-op rather than failing.
|
||||
- name: Run real-agent door tests
|
||||
run: |
|
||||
files=()
|
||||
for f in \
|
||||
test/e2e/bootstrap-real-claude.serial.test.ts \
|
||||
test/e2e/bootstrap-real-codex.serial.test.ts; do
|
||||
test/e2e/bootstrap-real-codex.serial.test.ts \
|
||||
test/e2e/install-real-hermes.serial.test.ts; do
|
||||
[ -f "$f" ] && files+=("$f")
|
||||
done
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
@@ -144,3 +150,187 @@ jobs:
|
||||
# --timeout: real-agent turns are slow (live claude/codex); the door
|
||||
# tests self-skip without authed binaries so this is a no-op elsewhere.
|
||||
bun test --timeout=600000 "${files[@]}"
|
||||
|
||||
# Hermes door e2e: unlike real-agent-e2e above (best-effort, self-skipping),
|
||||
# this job PROVISIONS the real hermes binary itself — pinned installer digest,
|
||||
# non-interactive auth + model pin — and then requires the door tests to
|
||||
# actually execute. It pays real API cost, so it stays label-gated at heavy
|
||||
# cadence (nightly + `real-agent-e2e`/`heavy-tests` label + dispatch); NEVER
|
||||
# the PR shard matrix. Loud-fail throughout: a runner that cannot install or
|
||||
# auth hermes fails this job rather than skipping.
|
||||
hermes-door:
|
||||
name: Hermes door e2e (real binary, loud-fail)
|
||||
if: |
|
||||
github.event_name != 'pull_request' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'real-agent-e2e') ||
|
||||
contains(github.event.pull_request.labels.*.name, 'heavy-tests')
|
||||
runs-on: ubuntu-latest
|
||||
# Four serial door tests at 600s each plus the installer budget cannot
|
||||
# fit the sibling job's 20 minutes.
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# Pin values documented in docs/mcp/HERMES-CLI-PIN.md — update them
|
||||
# together, deliberately, after reviewing upstream changes. The digest
|
||||
# pins the INSTALLER SCRIPT; the tag + commit pin the PAYLOAD it clones
|
||||
# (without them, the installer pulls upstream main into the runner that
|
||||
# later holds secrets). The commit is v2026.8.3's dereferenced SHA —
|
||||
# immutable even if the tag moves.
|
||||
HERMES_VERSION: "0.20.0"
|
||||
HERMES_GIT_TAG: "v2026.8.3"
|
||||
HERMES_GIT_COMMIT: "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
|
||||
HERMES_INSTALL_SHA256: "c118ff31618dc70339049ce71061b8f1351a1c70d9c2a236ed50d8a2550c550d"
|
||||
GBRAIN_REAL_HERMES_E2E: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
|
||||
# `runner.temp` is not an allowed context in job-level env, so the
|
||||
# evidence dir is derived here and exported for every later step (the
|
||||
# door test writes into it; the failure-path upload reads it — exporting
|
||||
# early keeps that upload working even when an install/precondition step
|
||||
# is what failed).
|
||||
- name: Prepare evidence dir
|
||||
run: |
|
||||
echo "GBRAIN_E2E_EVIDENCE_DIR=$RUNNER_TEMP/hermes-door-evidence" >> "$GITHUB_ENV"
|
||||
mkdir -p "$RUNNER_TEMP/hermes-door-evidence"
|
||||
|
||||
# NO secrets in this step's env: the installer is downloaded from the
|
||||
# network and executed, so it must never see credentials. The digest pin
|
||||
# turns an upstream installer change into a loud failure instead of
|
||||
# unreviewed code running in CI. Bound + retry the install run itself
|
||||
# (same posture as the e2e tier2 OpenClaw install): `timeout` kills a
|
||||
# hung attempt, 3 attempts ride out transient flakes, step cap backstops.
|
||||
- name: Install hermes (pinned installer digest)
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
curl -fsSL --retry 3 -o hermes-install.sh https://hermes-agent.nousresearch.com/install.sh
|
||||
if ! echo "$HERMES_INSTALL_SHA256 hermes-install.sh" | sha256sum -c -; then
|
||||
echo "::error::hermes installer digest drift — re-pin deliberately: update HERMES_INSTALL_SHA256 + HERMES_VERSION in this workflow and docs/mcp/HERMES-CLI-PIN.md after reviewing upstream changes" >&2
|
||||
exit 1
|
||||
fi
|
||||
for attempt in 1 2 3; do
|
||||
if timeout 600 bash hermes-install.sh --skip-setup --non-interactive --branch "$HERMES_GIT_TAG" --commit "$HERMES_GIT_COMMIT"; then
|
||||
# The branch/commit flags above are ASSERTED here, not trusted:
|
||||
# a shell installer that silently ignores unknown flags would
|
||||
# clone upstream main into a runner that later holds secrets.
|
||||
# Verify the actual checkout before anything else runs it.
|
||||
actual_commit=$(git -C "$HOME/.hermes/hermes-agent" rev-parse HEAD 2>/dev/null || echo "no-git-checkout")
|
||||
if [ "$actual_commit" != "$HERMES_GIT_COMMIT" ]; then
|
||||
echo "::error::hermes payload drift — installed checkout is $actual_commit, pinned $HERMES_GIT_COMMIT. Either the installer ignored its branch/commit flags or the layout moved from ~/.hermes/hermes-agent; re-pin deliberately (HERMES_GIT_TAG/HERMES_GIT_COMMIT + docs/mcp/HERMES-CLI-PIN.md) after reviewing upstream." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
exit 0
|
||||
fi
|
||||
echo "::warning::hermes install attempt $attempt failed or timed out; retrying in 10s" >&2
|
||||
sleep 10
|
||||
done
|
||||
echo "::error::hermes install failed after 3 attempts" >&2
|
||||
exit 1
|
||||
|
||||
- name: Preconditions (binary, secret, version pin)
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
run: |
|
||||
if ! command -v hermes >/dev/null 2>&1; then
|
||||
echo "::error::hermes did not resolve on PATH after install" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$ANTHROPIC_API_KEY" ]; then
|
||||
echo "::error::ANTHROPIC_API_KEY secret is empty — fork PRs get no secrets from GitHub, and this labeled job cannot run without them" >&2
|
||||
exit 1
|
||||
fi
|
||||
version_output=$(hermes --version)
|
||||
echo "$version_output"
|
||||
# Observed shape: `Hermes Agent v0.20.0 (2026.8.3)`.
|
||||
if ! printf '%s' "$version_output" | grep -qF "v$HERMES_VERSION"; then
|
||||
echo "::error::hermes version drift — expected v$HERMES_VERSION in: $version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Configure hermes (auth + model pin)
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
run: |
|
||||
mkdir -p ~/.hermes
|
||||
printf 'ANTHROPIC_API_KEY=%s\n' "$ANTHROPIC_API_KEY" > ~/.hermes/.env
|
||||
chmod 600 ~/.hermes/.env
|
||||
# `hermes model` is interactive-only; `config set` is the observed
|
||||
# non-interactive model pin.
|
||||
hermes config set model.default anthropic/claude-haiku-4.5
|
||||
# Global health check — informational only, never a gate here.
|
||||
hermes doctor || true
|
||||
|
||||
- name: Run hermes door tests
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
run: |
|
||||
# Redirect to a file, then tail — never pipe bun through tail (the
|
||||
# pipe eats the real exit code and truncates failure details).
|
||||
# `|| EXIT=$?` keeps the default `-e` shell from bailing before the
|
||||
# tail runs.
|
||||
EXIT=0
|
||||
bun test --timeout=600000 test/e2e/install-real-hermes.serial.test.ts > door.txt 2>&1 || EXIT=$?
|
||||
tail -40 door.txt
|
||||
if [ "$EXIT" -ne 0 ]; then
|
||||
# Preserve the FULL bun output for the failure artifact — bun
|
||||
# prints failure details before the summary, so the 40-line tail
|
||||
# above can drop exactly the lines a paid-CI triage needs.
|
||||
cp door.txt "$GBRAIN_E2E_EVIDENCE_DIR/" 2>/dev/null || true
|
||||
exit "$EXIT"
|
||||
fi
|
||||
# This job provisions the binary + auth above, so the door must
|
||||
# actually EXECUTE: a summary with zero passing tests means the
|
||||
# suite ran nothing or self-skipped everything — never let that
|
||||
# read as green.
|
||||
pass_count=$(grep -Eo '[0-9]+ pass' door.txt | tail -1 | grep -Eo '^[0-9]+' || true)
|
||||
if [ -z "$pass_count" ] || [ "$pass_count" -eq 0 ]; then
|
||||
echo "::error::hermes door summary shows no passing tests (nothing ran or everything skipped) — refusing to go green while testing nothing" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The door test copies its evidence into GBRAIN_E2E_EVIDENCE_DIR; the
|
||||
# workflow only uploads it. The test already excludes credential files —
|
||||
# the scrub below is defensive belt-and-suspenders before upload. Both
|
||||
# steps also require the evidence-dir env (a failure before the prepare
|
||||
# step leaves it unset, and there is nothing to upload then anyway).
|
||||
- name: Scrub credentials from evidence (defensive)
|
||||
if: failure() && env.GBRAIN_E2E_EVIDENCE_DIR != ''
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
run: |
|
||||
# Three layers, because the evidence dir carries files WRITTEN BY
|
||||
# THE THIRD-PARTY HERMES BINARY (logs/sessions) and upload-artifact
|
||||
# follows symlinks:
|
||||
# 1. filename scrub (env files),
|
||||
# 2. symlink delete (an agent-dropped link could dereference to a
|
||||
# real credential file at upload time),
|
||||
# 3. content scrub (any file that embeds the key — auth-error dumps
|
||||
# are most likely exactly on the failure path that uploads).
|
||||
find "$GBRAIN_E2E_EVIDENCE_DIR" -type f \( -name '.env' -o -name '*.env' \) -exec rm -f {} + 2>/dev/null || true
|
||||
find "$GBRAIN_E2E_EVIDENCE_DIR" -type l -delete 2>/dev/null || true
|
||||
if [ -n "$ANTHROPIC_API_KEY" ]; then
|
||||
grep -rlF "$ANTHROPIC_API_KEY" "$GBRAIN_E2E_EVIDENCE_DIR" 2>/dev/null | while IFS= read -r f; do
|
||||
echo "::warning::removing evidence file containing the API key: ${f#"$GBRAIN_E2E_EVIDENCE_DIR"/}" >&2
|
||||
rm -f "$f"
|
||||
done
|
||||
fi
|
||||
- name: Upload hermes door evidence
|
||||
if: failure() && env.GBRAIN_E2E_EVIDENCE_DIR != ''
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: hermes-door-evidence
|
||||
path: ${{ env.GBRAIN_E2E_EVIDENCE_DIR }}
|
||||
retention-days: 14
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Hosted ubuntu-latest runners are ephemeral, but this must not depend
|
||||
# on that: if the job ever moves to a self-hosted runner (the sibling
|
||||
# real-agent-e2e job is designed for one), a key left in ~/.hermes/.env
|
||||
# would persist for every later workload on that runner.
|
||||
- name: Remove hermes credentials (unconditional)
|
||||
if: always()
|
||||
run: rm -f ~/.hermes/.env
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<!-- gbrain-runbook-stamp: 0.45.11.0 -->
|
||||
<!-- gbrain-runbook-stamp: 0.45.12.0 -->
|
||||
<!-- This stamp must equal the VERSION file at every release; CI enforces it
|
||||
(scripts/check-bootstrap-tag.sh). `gbrain bootstrap status` compares it to
|
||||
the installed binary and warns on skew. -->
|
||||
|
||||
@@ -2,6 +2,68 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.45.12.0] - 2026-08-13
|
||||
|
||||
**Hermes joins the tested-install club: a real-binary harness now proves gbrain works inside Hermes, and `gbrain friction diff` tells you whether an install problem is the agent's or ours.**
|
||||
|
||||
GBrain has long said "works with OpenClaw and Hermes." For OpenClaw that claim was
|
||||
backed by tests; for Hermes it was backed by hope. This release closes that gap with a
|
||||
full end-to-end harness: a Hermes runner for the claw-test friction lab, a real-binary
|
||||
"door" test that registers gbrain into an actual Hermes install over MCP and asks it to
|
||||
recall a seeded fact, and a CI job that installs a pinned Hermes release and runs the
|
||||
door on demand. Every Hermes CLI behavior the harness relies on was pinned by observing
|
||||
a real install — the flag-order traps, the interactive prompts, the exit-code quirks —
|
||||
and those observations ship as documentation so your own Hermes setup benefits too.
|
||||
|
||||
The live claw-test lane also got honest: it now stages the scenario workspace before the
|
||||
agent starts and verifies real outcomes after it finishes, so an agent that does nothing
|
||||
and exits cleanly finally FAILS the run instead of passing it. And with two runners in
|
||||
the registry, the new `gbrain friction diff --base openclaw --compare hermes` turns
|
||||
friction reports into a comparison instrument: pain unique to one agent is that agent's
|
||||
contract problem; pain common to both is ours.
|
||||
|
||||
## To take advantage of v0.45.12.0
|
||||
|
||||
`gbrain upgrade` is enough — no schema migration.
|
||||
|
||||
1. **Running Hermes?** Wire gbrain in with one command (full guide at
|
||||
`docs/mcp/HERMES.md`, including the non-obvious flag-order and prompt gotchas):
|
||||
```bash
|
||||
printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve
|
||||
hermes mcp test gbrain
|
||||
```
|
||||
2. **Want the friction lab on your own agent?**
|
||||
```bash
|
||||
gbrain claw-test --live --agent hermes # or --agent openclaw
|
||||
gbrain friction diff --base openclaw --compare hermes
|
||||
```
|
||||
3. **If anything looks wrong,** file an issue at https://github.com/garrytan/gbrain/issues
|
||||
with `gbrain doctor` output.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
**Added**
|
||||
- Hermes runner for the claw-test harness: `gbrain claw-test --live --agent hermes` drives a real Hermes install headlessly (`$HERMES_BIN` override supported; `--list-agents` shows availability for both runners).
|
||||
- `gbrain friction diff --base <run-or-agent> --compare <run-or-agent>`: cross-agent friction comparison with unique-to-each and changed sections, count deltas, and a compatibility banner that warns when runs cover different scenarios or versions. `--json` for machines.
|
||||
- Real-binary Hermes install door e2e (`test/e2e/install-real-hermes.serial.test.ts`): registers this checkout's gbrain into a hermetic Hermes home via a live MCP handshake (110 tools discovered), verifies both the CLI and direct-config registration surfaces, and proves recall of a seeded fact in a paid one-shot turn. Triple-gated so it can never burn tokens by accident.
|
||||
- Label-gated `hermes-door` CI job: installs a digest-and-tag-pinned Hermes release, refuses to go green if nothing actually ran, and uploads scrubbed evidence on failure.
|
||||
- Per-client MCP docs for Hermes (`docs/mcp/HERMES.md`) and OpenClaw (`docs/mcp/OPENCLAW.md`), plus a dev-facing pin of every observed Hermes CLI behavior (`docs/mcp/HERMES-CLI-PIN.md`) and an install snippet in `INSTALL_FOR_AGENTS.md`.
|
||||
- Generic agent-workspace compatibility test pinning the documented "any repo with a workspace" install flow (detection, scaffold additivity, resolver health).
|
||||
|
||||
**Changed**
|
||||
- claw-test live mode now stages the scenario before the agent runs (fresh-install: brain pages + routing file + init; upgrade: seed replay) and verifies outcomes after it exits — doctor health, a scenario-declared query returning results, expected files existing, and for upgrades a non-mutating schema-version probe that a do-nothing agent cannot satisfy.
|
||||
- The brief handed to live agents now matches the current CLI exactly (extract argument shape, doctor status vocabulary), and bare `gbrain` inside a live run resolves to the harness's own binary via a per-run PATH shim.
|
||||
- Every claw-test run opens and closes with a machine-readable marker carrying the agent name and scenario, so friction analytics can resolve runs by agent; scripted runs are now labeled `scripted` instead of borrowing an agent's name.
|
||||
- Scenario oracle configuration is validated on load — misdeclared oracles fail loudly instead of silently not being enforced.
|
||||
|
||||
**Fixed**
|
||||
- claw-test now works out of the box when gbrain runs from source (`bun run src/cli.ts`): child invocations resolve to a real gbrain launcher instead of the bun runtime itself, which previously made the default harness unusable outside compiled builds.
|
||||
- Upgrade-scenario runs in BOTH modes fail loudly when the scenario ships no seed dump, instead of quietly initializing a current database and reporting an "upgrade" that never exercised a migration.
|
||||
- Every harness child process now runs under a wall-clock timeout, and live-agent timeouts kill the agent's whole process tree — a hung child no longer wedges a run (or a CI job) forever.
|
||||
- Agent-side friction entries now survive the run's tempdir cleanup: they merge into your friction log before the workspace is deleted, so `friction render` and `friction diff` finally see both halves of a live run.
|
||||
- `claw-test --list-agents` no longer races CLI teardown; output is complete and ordered.
|
||||
- Live runs keep the agent's gbrain children pointed at the run's own hermetic brain even when the surrounding shell exports a database-pointing environment variable — the harness's verification and the agent's work can no longer land in two different places.
|
||||
- The test real-name guard now correctly distinguishes the public Hermes platform (documented and tested) from private deployment names (still banned).
|
||||
## [0.45.11.0] - 2026-08-12
|
||||
|
||||
**The install now ends by telling you the two things that matter: you own the brain, and here's the first skill to run.** A working install used to finish on a health report and three tour prompts — technically complete, but a new user walked away without the two facts that make gbrain worth trusting and worth using. Now `gbrain bootstrap verify` ends with a hand-off: **what you own** (every memory is a markdown file in YOUR private GitHub repo — read it, take it to a second machine, delete it and the brain is gone; or the local-only variant with the one command that gives it a durable home) and **what to do next** (run the `cold-start` skill — say "fill my brain" and your agent imports your Gmail, calendar, and contacts through ClawVisor, an OAuth vault so the agent never holds raw tokens, or offline archives like Google Takeout, one consented phase at a time).
|
||||
|
||||
@@ -506,7 +506,7 @@ four numeric segments are required first. Historical 3-segment versions
|
||||
| `CHANGELOG.md` | Top entry header `## [0.31.4.1] - YYYY-MM-DD` plus the "To take advantage of v0.31.4.1" block. | Standard Keep-a-Changelog header. |
|
||||
| `TODOS.md` | Any TODO entries that mention "follow-up from vX.Y.Z.W" use the version of the release that filed them. Update only when filing NEW follow-up TODOs. | Inline `vX.Y.Z.W` references in TODO bodies. |
|
||||
| `CLAUDE.md` | The Key Files section's per-file annotations carry `vX.Y.Z.W (#NNN)` tags noting which release introduced a behavior. Update whenever a wave's annotations get folded in. | Inline `vX.Y.Z.W (#NNN, contributed by @user)` references. |
|
||||
| `openclaw.plugin.json` | OpenClaw plugin manifest (v0.45.6.0, #4033). Hand-maintained; `test/openclaw-plugin-manifest.test.ts` fails the suite if it drifts from `package.json`. Merges from master auto-resolve it to master's version — re-bump it with the trio. | `"version": "0.45.10.0"` |
|
||||
| `openclaw.plugin.json` | OpenClaw plugin manifest (v0.45.6.0, #4033). Hand-maintained; `test/openclaw-plugin-manifest.test.ts` fails the suite if it drifts from `package.json`. Merges from master auto-resolve it to master's version — re-bump it with the trio. | `"version": "0.45.12.0"` |
|
||||
| `BOOTSTRAP_FOR_AGENTS.md` | Runbook stamp on line 1. `scripts/check-bootstrap-tag.sh` (in `bun run verify` + CI) fails when it drifts from `VERSION`; refresh it in the same commit as the bump. | `<!-- gbrain-runbook-stamp: X.Y.Z.W -->` |
|
||||
| `templates/bootstrap/template-repo/` | Vendored template tree with an embedded version stamp. Auto-derived, but NOT by `bun install`: run `bun run scripts/generate-template-repo.ts --out templates/bootstrap/template-repo` after the bump; `scripts/check-bootstrap-templates.sh` fails CI on drift. | `<!-- gbrain-template-stamp: X.Y.Z.W -->` in generated files. |
|
||||
|
||||
|
||||
@@ -217,6 +217,16 @@ diff against gbrain's bundle when you want upstream improvements. (The legacy
|
||||
`gbrain skillpack install` managed-block model was removed in v0.33 — run
|
||||
`gbrain skillpack migrate-fence` once if upgrading from an older release.)
|
||||
|
||||
**If you are Hermes:** register gbrain as your MCP server:
|
||||
|
||||
```bash
|
||||
printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve
|
||||
```
|
||||
|
||||
Keep `--args` last (everything after it becomes server argv) and verify with
|
||||
`hermes mcp test gbrain` — the add exits 0 even on failure. Full reference:
|
||||
[docs/mcp/HERMES.md](docs/mcp/HERMES.md).
|
||||
|
||||
Whether you scaffolded or not, read `skills/RESOLVER.md` (in your workspace, or the
|
||||
bundled copy at `~/gbrain/skills/RESOLVER.md` when running from the cloned repo). It's
|
||||
the skill dispatcher — tells you which skill to read for any task. Save this to your
|
||||
|
||||
@@ -172,6 +172,8 @@ GBrain exposes nearly all of its 100+ operations as MCP tools (stdio and HTTP; a
|
||||
- **[Claude Code](docs/mcp/CLAUDE_CODE.md)** — local: one command, `claude mcp add gbrain -- gbrain serve` (zero server, zero tunnel). Remote with just a bearer token: `gbrain connect https://your-host/mcp --token gbrain_xxx` prints a paste-ready block (or `--install` wires it up and smoke-tests the token).
|
||||
- **[Codex](docs/mcp/CODEX.md)** — `gbrain connect https://your-host/mcp --token gbrain_xxx --agent codex` (or `--install`). Codex reads the bearer from `$GBRAIN_REMOTE_TOKEN` at runtime, so the token never lands in Codex config.
|
||||
- **[Cursor / Windsurf / any stdio MCP client](docs/mcp/CLAUDE_CODE.md)** — same shape, add `{"command": "gbrain", "args": ["serve"]}` to your MCP config.
|
||||
- **[Hermes](docs/mcp/HERMES.md)** — `printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve`. Keep `--args` last, and verify with `hermes mcp test gbrain` (the add exits 0 even on failure).
|
||||
- **[OpenClaw](docs/mcp/OPENCLAW.md)** — the ClawHub bundle plugin registers gbrain automatically (`openclaw.plugin.json` ships in this repo), or add `{"command": "gbrain", "args": ["serve"]}` to `~/.openclaw/config.json`'s `mcpServers`.
|
||||
- **[Claude Desktop (Cowork)](docs/mcp/CLAUDE_DESKTOP.md)** — Settings → Integrations → add the URL of your HTTP server. Remote only; the local `claude_desktop_config.json` does not work for remote servers.
|
||||
- **[Claude Cowork (team plan)](docs/mcp/CLAUDE_COWORK.md)** — org Owner adds the connector under Organization Settings → Connectors.
|
||||
- **[Perplexity Computer](docs/mcp/PERPLEXITY.md)** — `gbrain connect https://your-host/mcp --agent perplexity --oauth --register` mints a least-privilege OAuth client and prints the Issuer/Client ID/Secret to paste into Settings → Connectors (OAuth is the right path for a cloud connector; a bearer token also works for local use). Pro subscription required.
|
||||
|
||||
@@ -3840,28 +3840,99 @@ After the sweep, both should be fixable and renameable back to plain `*.test.ts`
|
||||
|
||||
## claw-test E2E (v0.22.16 follow-ups)
|
||||
|
||||
### Hermes runner — `src/core/claw-test/runners/hermes.ts`
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Add a Hermes implementation of the `AgentRunner` interface. v1 ships only OpenClaw; v1.1 lands hermes once we have real friction reports from openclaw to validate the contract against.
|
||||
|
||||
**Why:** Cross-agent diff (`gbrain friction diff --base openclaw --compare hermes`) is the highest-leverage next signal. Friction unique to one agent vs common-to-both separates "agent contract bug" from "gbrain bug" automatically.
|
||||
|
||||
**Effort:** S (CC ~30m). Depends on: v1 openclaw runner producing real friction reports first.
|
||||
### ~~Hermes runner — `src/core/claw-test/runners/hermes.ts`~~ DONE (hermes-harness wave)
|
||||
Shipped: `HermesRunner` (`hermes -z <brief>`, `$HERMES_BIN` > `which hermes`,
|
||||
`HERMES_HOME` env-allowlist delta) + the full hermes install door
|
||||
(`test/e2e/install-real-hermes.serial.test.ts`, opt-in-gated) + the label-gated
|
||||
`hermes-door` CI job in heavy-tests.yml. The cross-agent
|
||||
`gbrain friction diff --base openclaw --compare hermes` payoff shipped in the
|
||||
same wave (below). Observed-CLI pins live in `docs/mcp/HERMES-CLI-PIN.md` and
|
||||
`docs/mcp/HERMES.md`.
|
||||
|
||||
---
|
||||
|
||||
### Friction analytics suite — `diff` / `trend` / `migration-stub`
|
||||
### Friction analytics suite — `trend` / `migration-stub` (diff SHIPPED)
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Three new `gbrain friction` subcommands deferred from v1:
|
||||
- `gbrain friction diff --base <run-or-agent> --compare <run-or-agent>` (cross-agent comparison; ~80 LOC)
|
||||
**What:** Two remaining `gbrain friction` subcommands deferred from v1
|
||||
(`diff` shipped in the hermes-harness wave — see `src/commands/friction.ts`):
|
||||
- `gbrain friction trend [--since <version-or-date>] [--phase <name>]` (time-series across runs; ~60 LOC)
|
||||
- `gbrain friction migration-stub [--threshold N]` (clusters friction by phase + tokens, emits `skills/migrations/v[N+1].md` stub; ~150 LOC)
|
||||
|
||||
**Why:** Turns point-in-time reports into a slope. Pairs with the v1.1 public scoreboard.
|
||||
|
||||
**Effort:** M (CC ~2h total).
|
||||
**Effort:** M (CC ~1.5h total).
|
||||
|
||||
---
|
||||
|
||||
### Promote hermes-door soft probes to hard assertions + build the REAL cron test
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Two follow-ups now that the hermes CLI surface is pinned (v0.20.0,
|
||||
`docs/mcp/HERMES-CLI-PIN.md`): (1) promote the door's logged-evidence probes
|
||||
(`hermes mcp list` output shape; session-artifact tool-call traces under
|
||||
`<home>/.hermes/`) to hard assertions once a couple of CI runs confirm their
|
||||
stability across hermes releases; (2) build the real cron pairing test — the
|
||||
surface is fully non-interactive (`hermes cron create [--name N] [--no-agent]
|
||||
[--script PATH] <schedule> [prompt]` + `hermes cron tick` runs due jobs once
|
||||
and exits) — create a job that runs `gbrain sync --json`, tick, and assert the
|
||||
sync actually executed against the run's brain. (A self-skipping probe was
|
||||
deliberately CUT in review: a test that cannot fail is not coverage.)
|
||||
|
||||
**Why:** INSTALL_FOR_AGENTS.md's recurring-jobs step has zero coverage; the
|
||||
evidence sweep is the promotion signal the door already logs.
|
||||
|
||||
**Effort:** S-M (CC ~45m). Depends on: first labeled hermes-door CI runs.
|
||||
|
||||
---
|
||||
|
||||
### Wire the orphaned `voice-agent-install` ScenarioKind
|
||||
**Priority:** P2
|
||||
|
||||
**What:** `test/fixtures/claw-test-scenarios/voice-agent-install/` carries the
|
||||
richest install-assertion template in the repo (60-line expected.json:
|
||||
filesystem manifest, `.gbrain-source.json` sha256s, resolver rows, PII
|
||||
blocklist, health probe, tiered soft-fail) but `scenario.json` declares
|
||||
`kind: "voice-agent-install"`, which `ScenarioKind` rejects — the fixture
|
||||
cannot load. Extend `ScenarioKind` + `loadScenario` + a `postInstallHook`
|
||||
implementation so the scenario runs.
|
||||
|
||||
**Why:** Integrations-recipe install coverage (the `gbrain integrations
|
||||
install` path) has a fully-designed scenario sitting dead.
|
||||
|
||||
**Effort:** M (CC ~1h). Integrations-lane work, deliberately kept out of the
|
||||
hermes-harness wave.
|
||||
|
||||
---
|
||||
|
||||
### Cold-install container test — fill the `tests/docker/bootstrap-e2e.sh` placeholder
|
||||
**Priority:** P3
|
||||
|
||||
**What:** heavy-tests.yml carries a gated no-op step for
|
||||
`tests/docker/bootstrap-e2e.sh` (networkless cold-machine container install of
|
||||
gbrain itself: global install, PATH discovery, migrations). The file doesn't
|
||||
exist. Write it.
|
||||
|
||||
**Why:** The agent-platform door tests (claude/codex/hermes) all deliberately
|
||||
run gbrain from the dev tree / compiled binary — none of them proves gbrain's
|
||||
own cold install. That gap was re-flagged in the hermes-harness wave's outside
|
||||
review and scoped OUT of that wave on purpose.
|
||||
|
||||
**Effort:** M (CC ~1-2h, docker).
|
||||
|
||||
---
|
||||
|
||||
### BrainBench hermes adapter
|
||||
**Priority:** P3
|
||||
|
||||
**What:** ~50-100 lines in `src/eval/brainbench/adapters/hermes.ts` + an
|
||||
`ALL_HARNESSES` entry + baseline cells in `evals/brainbench/baselines/main.json`.
|
||||
|
||||
**Why:** Cross-harness memory-conformance coverage for the third platform.
|
||||
Eval seam (memory conformance), NOT install — kept out of the install wave on
|
||||
purpose; needs baseline-governance care per the BrainBench gate rules.
|
||||
|
||||
**Effort:** S-M (CC ~1h + baseline runs).
|
||||
|
||||
---
|
||||
|
||||
@@ -3881,7 +3952,7 @@ After the sweep, both should be fixable and renameable back to plain `*.test.ts`
|
||||
### Real v0.18 SQL dump for upgrade scenario
|
||||
**Priority:** P2
|
||||
|
||||
**What:** The `upgrade-from-v0.18` scenario ships scaffolded — `seed/dump.sql` is missing. The harness gracefully no-ops the seed phase when absent, so the scenario currently behaves like fresh-install. v1.1: generate a real v0.18-shape PGLite dump per the procedure documented in `test/fixtures/claw-test-scenarios/upgrade-from-v0.18/seed/README.md`.
|
||||
**What:** The `upgrade-from-v0.18` scenario ships scaffolded — `seed/dump.sql` is missing. Both scripted and live runs now FAIL LOUDLY on the missing dump (a silent skip used to init a current database and false-green the "upgrade"), so the shipped scenario is unrunnable until the dump lands. Generate a real v0.18-shape PGLite dump per the procedure documented in `test/fixtures/claw-test-scenarios/upgrade-from-v0.18/seed/README.md`.
|
||||
|
||||
**Why:** Without a real seed, the scenario doesn't actually exercise the migration chain forward-walk. That's the whole point of the upgrade scenario — proves issue #239/#243/#266/#357 class regressions stay fixed.
|
||||
|
||||
|
||||
+4
-2
@@ -13,7 +13,7 @@ Mix later if needed.
|
||||
|
||||
## 1. Run with an agent platform
|
||||
|
||||
Already running [OpenClaw](https://github.com/garrytan/openclaw) or [Hermes](https://github.com/garrytan/hermes)?
|
||||
Already running [OpenClaw](https://github.com/garrytan/openclaw) or [Hermes](https://github.com/NousResearch/hermes-agent)?
|
||||
|
||||
```bash
|
||||
bun install -g github:garrytan/gbrain#latest-stable
|
||||
@@ -71,7 +71,7 @@ claude mcp add gbrain -- gbrain serve --surface verbs # Claude Code
|
||||
codex mcp add gbrain -- gbrain serve --surface verbs # Codex
|
||||
```
|
||||
|
||||
The agent spawns `gbrain serve` as a stdio subprocess against your local brain. `--surface verbs` gives the agent the five-verb memory protocol (`recall`, `remember`, `entity`, `synthesize`, `forget` — [MEMORY_VERBS v1](protocol/MEMORY_VERBS_v1.md)) instead of the full tool catalog; drop the flag (default `full`) for every operation. Full walkthrough (both this local path and connecting to a remote brain), plus the brain-first protocol to paste into `CLAUDE.md` / `AGENTS.md`: **[Give your coding agent a memory](tutorials/connect-coding-agent.md)**.
|
||||
The agent spawns `gbrain serve` as a stdio subprocess against your local brain. `--surface verbs` gives the agent the seven-verb memory protocol (`recall`, `remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` — [MEMORY_VERBS v1](protocol/MEMORY_VERBS_v1.md)) instead of the full tool catalog; drop the flag (default `full`) for every operation. Full walkthrough (both this local path and connecting to a remote brain), plus the brain-first protocol to paste into `CLAUDE.md` / `AGENTS.md`: **[Give your coding agent a memory](tutorials/connect-coding-agent.md)**.
|
||||
|
||||
## 3. MCP server (any MCP client)
|
||||
|
||||
@@ -98,6 +98,8 @@ Per-client setup guides live in [`docs/mcp/`](mcp/):
|
||||
- [`docs/mcp/CODEX.md`](mcp/CODEX.md)
|
||||
- [`docs/mcp/CLAUDE_DESKTOP.md`](mcp/CLAUDE_DESKTOP.md)
|
||||
- [`docs/mcp/CHATGPT.md`](mcp/CHATGPT.md)
|
||||
- [`docs/mcp/HERMES.md`](mcp/HERMES.md)
|
||||
- [`docs/mcp/OPENCLAW.md`](mcp/OPENCLAW.md)
|
||||
- [`docs/mcp/PERPLEXITY.md`](mcp/PERPLEXITY.md)
|
||||
- [`docs/mcp/DEPLOY.md`](mcp/DEPLOY.md) — production deploy patterns
|
||||
|
||||
|
||||
+4
-1
@@ -282,7 +282,9 @@ E2E tests live in `test/e2e/` and run against real Postgres+pgvector (require `D
|
||||
- `test/e2e/pglite-cli-exit.serial.test.ts` — real spawned-CLI exit behavior on PGLite (in-memory, no `DATABASE_URL`): read commands (`search`/`get`/`query`) exit 0 promptly; CLI_ONLY `capture` exits clean and frees the single-writer lock; the `#2084` describes pin every swept disconnect site — a failed op exits 1 with the error on stderr, and the dashboard, read-only-timeout, doctor, and `dream --dry-run` paths all exit with no force-exit banner.
|
||||
- `test/e2e/pgbouncer-teardown.test.ts` — PgBouncer TRANSACTION-mode teardown (#2084 / the #1972→#2015→#2084 class). Pins the bug CLASS, not timings: a CLI op against a txn-mode pooled URL exits 0 with intact stdout and does NOT ride the 10s hard-deadline backstop (the `engine.disconnect() did not return` banner is the smoking gun — pre-#2084 it printed on 100% of query-shaped ops). Gated by `GBRAIN_PGBOUNCER_URL` + `GBRAIN_PGBOUNCER_DIRECT_URL` (NOT `DATABASE_URL`) — set automatically by `bun run ci:local`'s `pgbouncer` compose service; skips gracefully elsewhere. Uses a DEDICATED `gbrain_pgbouncer` database so it never races the `gbrain_test` TRUNCATE fixtures.
|
||||
- `test/e2e/volunteer-context-postgres.test.ts` — `volunteer_context` on REAL Postgres (#2095; engine parity beyond the hermetic PGLite unit suite): resolution arms through the actual op handler, the fire-and-forget volunteer-event sink landing rows, the stats join, and the RLS pin that `context_volunteer_events` has ROW LEVEL SECURITY enabled (keeps the v35 auto-RLS event trigger honest for migration-created tables). `DATABASE_URL`-gated.
|
||||
- `test/e2e/openclaw-reference-compat.test.ts` — `check-resolvable` + `skillpack install` against a minimal AGENTS.md workspace fixture (`test/fixtures/openclaw-reference-minimal/`), regression guard for the OpenClaw deployment shape.
|
||||
- `test/e2e/openclaw-reference-compat.test.ts` — `check-resolvable` + skillpack install-model against a minimal AGENTS.md workspace fixture (`test/fixtures/openclaw-reference-minimal/`), regression guard for the OpenClaw deployment shape.
|
||||
- `test/e2e/workspace-generic-compat.test.ts` — always-on (PGLite, no binary): pins the INSTALL_FOR_AGENTS.md "any repo with a workspace" contract against `test/fixtures/generic-agents-workspace/` (Hermes is the motivating consumer): `cwd_walk_up` detection, the `GBRAIN_SKILLS_DIR` override, `check-resolvable` on a root AGENTS.md, and scaffold additivity + refuse-overwrite. The real Hermes-behavior proof is the door suite below.
|
||||
- `test/e2e/install-real-hermes.serial.test.ts` — the hermes "door": real `hermes` binary + real `hermes mcp add` handshake (full-catalog tool discovery; the count tracks the op catalog, so the test asserts discovery happened, not a number) + a paid `hermes -z` recall turn against a seeded brain. Triple-gated: `GBRAIN_REAL_HERMES_E2E=1` (explicit opt-in — run-e2e.sh scrubs GBRAIN_*, so it can never fire under `bun run test:e2e`) + resolvable binary + non-empty ANTHROPIC key (anthropic-pinned on purpose: a second provider key flips hermes provider-auto into a mis-routed 401). Hermetic HOME + HERMES_HOME with a tripwire on the operator's real config; evidence copies to `GBRAIN_E2E_EVIDENCE_DIR` for CI upload. Venue: heavy-tests.yml (`real-agent-e2e` + `hermes-door` jobs).
|
||||
- `test/e2e/search-swamp.test.ts` — reproduces the source-swamp case. Seeds a curated `originals/talks/article-outline-fat-code` page against two `<fork>/chat/` pages stuffed with the same multi-word phrase. Asserts the article wins keyword AND vector ranking, that `detail=high` lets the chat swamp re-surface, and that `source_id` passes through the two-stage CTE intact. PGLite in-memory.
|
||||
- `test/e2e/search-exclude.test.ts` — `test/` + `archive/` pages hidden by default, `include_slug_prefixes` opts back in, caller-supplied `exclude_slug_prefixes` adds to defaults. Both keyword and vector search paths.
|
||||
- `test/e2e/engine-parity.test.ts` — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector` (Postgres ranks pages then picks best chunk while PGLite returns chunks directly, so the source-boost behavior needs parity coverage). Skips without `DATABASE_URL`.
|
||||
@@ -297,6 +299,7 @@ E2E tests live in `test/e2e/` and run against real Postgres+pgvector (require `D
|
||||
- `test/e2e/think-source-isolation-pglite.test.ts` — PGLite in-memory suite pinning the `think` gather stage's source scope: seeds three sources with cross-source links and embedded takes, then asserts `runGather` under a federated `sourceIds` grant (and under a scalar `sourceId`) keeps every stream — hybrid retrieval, takes keyword + vector (`searchTakes`/`searchTakesVector`), and the `traversePaths` graph walk — inside the grant while still reaching authorized neighboring sources. No `DATABASE_URL` needed.
|
||||
- `test/e2e/skill-brain-first.test.ts` — doctor reports `skill_brain_first` check with structured issues; `--fix --dry-run` previews insertion without writing; `--fix` applies the canonical Convention callout idempotently; `brain_first: exempt` frontmatter resolves the warn; `brain_first_typo` surfaces a paste-ready hint; audit JSONL records `detected` / `resolved` / `fixed` transitions; stable brain emits 0 audit lines/run.
|
||||
- Tier 2 (`test/e2e/skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI.
|
||||
- `test/e2e/claw-test.test.ts` also covers live mode token-free via shim agents (`OPENCLAW_BIN=<sh script>`): the success-oracle break path (a do-nothing agent now FAILS), the E0 child-friction merge surviving tempdir cleanup, and the upgrade staging + schema-version-probe regression.
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees: `find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
- **Run E2E tests without asking permission.** When you want to verify behavior, there's a relevant E2E test, or you're shipping anything covered by an E2E suite — spin up the test DB, run the tests, tear down. Don't ask, don't propose it, don't defer. The lifecycle is short (~2-30s startup, sub-minute tests, instant teardown) and the gate value is high. Skipping with "DATABASE_URL unset" is silent regression, not caution.
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -20,8 +20,8 @@ claude mcp add gbrain -- gbrain serve --surface verbs
|
||||
That's it. Claude Code spawns `gbrain serve` as a stdio subprocess. No server, no
|
||||
tunnel, no token needed. Works with both PGLite and Supabase engines.
|
||||
|
||||
`--surface verbs` exposes the five-verb memory protocol (`recall`, `remember`,
|
||||
`entity`, `synthesize`, `forget` — [MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)),
|
||||
`--surface verbs` exposes the seven-verb memory protocol (`recall`, `remember`,
|
||||
`entity`, `synthesize`, `forget`, `context_pack`, `delta` — [MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)),
|
||||
the surface built for agents and quickstarts. Drop the flag for the full
|
||||
operation catalog (`get_page`, `put_page`, `search`, graph ops, …) — `full` is
|
||||
the default and what existing installs already run.
|
||||
|
||||
+2
-2
@@ -24,8 +24,8 @@ gbrain serve --surface verbs # just the 7 memory verbs (quickstart surface)
|
||||
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
`--surface verbs` exposes exactly the five-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget` —
|
||||
`--surface verbs` exposes exactly the seven-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)) instead of the full catalog;
|
||||
omit the flag (default `full`) for every operation.
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# Hermes CLI pin — observed behavior notes (v0.20.0)
|
||||
|
||||
Dev-facing companion to [HERMES.md](HERMES.md): every fact below was OBSERVED
|
||||
against a real install (2026-08-12), not researched from docs. The claw-test
|
||||
HermesRunner, the install door e2e, and the heavy-tests hermes-door CI job
|
||||
assert exactly these shapes — when hermes releases change them, update this
|
||||
file, the workflow pins, and the affected assertions together.
|
||||
|
||||
## Pin
|
||||
- **Hermes Agent v0.20.0 (2026.8.3)**, observed against git checkout `3e09adb` at
|
||||
`~/.hermes/hermes-agent` (an upstream-main commit carrying the same v0.20.0/2026.8.3
|
||||
version stamp; CI installs the RELEASE TAG `v2026.8.3` = commit `3c27eb62` — the two
|
||||
differ by post-release main commits, same declared version. If a CI door run ever
|
||||
diverges from these notes, re-observe against the tag checkout.)
|
||||
- Installer sha256: `c118ff31618dc70339049ce71061b8f1351a1c70d9c2a236ed50d8a2550c550d`
|
||||
(download https://hermes-agent.nousresearch.com/install.sh to a file first; verify; then run)
|
||||
- Installer flags used: `--skip-setup --non-interactive`; binary lands at `~/.local/bin/hermes`
|
||||
- Python 3.11.15 via uv
|
||||
|
||||
## HERMES_HOME — HONORED (verified)
|
||||
Installer (`HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"`) AND runtime both honor it:
|
||||
`mcp add`/`mcp list`/`config set` under `HERMES_HOME=<tmp>` read+write `<tmp>/config.yaml`,
|
||||
populate `<tmp>/{SOUL.md,cron,logs,...}`, and do NOT touch `~/.hermes`. Belt-and-suspenders
|
||||
(HOME + HERMES_HOME both to tmp) stays in the door test anyway.
|
||||
|
||||
## One-shot (`-z`)
|
||||
- `hermes -z "<prompt>"` → **stdout = final text ONLY**; benign notices may appear on stderr
|
||||
("Shell cwd was reset to ..."). Verified reply fidelity ("B0-PROBE-OK").
|
||||
- Exit codes: 0 = success; **1 = no inference provider configured** (message: "agent failed:
|
||||
No inference provider configured. Run 'hermes model' ... or set an API key
|
||||
(OPENROUTER_API_KEY, OPENAI_API_KEY, etc.) in ~/.hermes/.env.")
|
||||
- `--usage-file PATH` exists; per-call `-m MODEL --provider PROVIDER` exist; also
|
||||
`--in DIR`, `--ignore-user-config`, `--safe-mode`, `-t TOOLSETS`, `--skills`.
|
||||
|
||||
## Auth + model pin (non-interactive)
|
||||
- `$HERMES_HOME/.env` with `ANTHROPIC_API_KEY=...` WORKS (verified end-to-end).
|
||||
- Model pin: `hermes config set model.default anthropic/claude-haiku-4.5` → exit 0,
|
||||
writes `model.default` into config.yaml. `hermes config get model.default` reads it back.
|
||||
(`hermes model` is INTERACTIVE-only — never use it in tests/CI.)
|
||||
- Valid model id format: `anthropic/claude-haiku-4.5` (hermes catalog naming, provider-prefixed).
|
||||
|
||||
## `hermes mcp add` — THE big observed facts
|
||||
- Shape: `hermes mcp add <name> [--env K=V K2=V2 ...] [--connect-timeout N] --command CMD --args ...`
|
||||
**`--args` MUST be the last option** — anything after it (incl. a misplaced `--env`) is
|
||||
swallowed into the server argv. (First rehearsal failed exactly this way.)
|
||||
**The env flag takes MULTIPLE KEY=VALUE values after ONE flag; REPEATING it REPLACES the
|
||||
first occurrence** (argparse nargs semantics) — a repeated-flag invocation silently drops
|
||||
the earlier vars, the handshake fails, and the piped Y then hits the save-anyway prompt →
|
||||
the entry is saved with `enabled: false`. (First real door run failed exactly this way.)
|
||||
- Add performs a REAL MCP handshake + tool discovery at add time. Against
|
||||
`--command bun --args run <abs>/src/cli.ts serve` with `--env GBRAIN_HOME=<tmp>`:
|
||||
connected, discovered **110 gbrain tools**.
|
||||
- On success it prompts `Enable all N tools? [Y/n/select]:` — **non-interactive: pipe
|
||||
`printf 'Y\n'`**. Piping Y saves: `✓ Saved 'gbrain' to <HERMES_HOME>/config.yaml (110/110
|
||||
tools enabled)`. EOF on the prompt = `Cancelled.`, nothing saved.
|
||||
- **EXIT CODE IS 0 EVEN ON CONNECTION FAILURE OR CANCEL.** Never assert on `mcp add`'s exit
|
||||
code. Hard assertions = (a) `config.yaml` contains `mcp_servers.<name>` after the add,
|
||||
(b) `hermes mcp test <name>` exits 0.
|
||||
|
||||
## Saved config schema (verbatim shape)
|
||||
```yaml
|
||||
_config_version: 34
|
||||
mcp_servers:
|
||||
gbrain:
|
||||
command: bun
|
||||
args:
|
||||
- run
|
||||
- /abs/path/src/cli.ts
|
||||
- serve
|
||||
env:
|
||||
GBRAIN_HOME: /tmp/gb-xxxx
|
||||
connect_timeout: 60.0
|
||||
enabled: true
|
||||
```
|
||||
(The generated file also contains commented template blocks — security, fallback_model.)
|
||||
|
||||
## Probes
|
||||
- `hermes mcp list` → table `Name / Transport / Tools / Status`, row shows `gbrain ... ✓ enabled`.
|
||||
- `hermes mcp test gbrain` → exit 0 + prints the tool list. THE targeted probe for Test 1b.
|
||||
- `hermes doctor` exists (global health; not a per-server assertion).
|
||||
|
||||
## Cron (for the post-pin F7 TODO — real test is buildable)
|
||||
`hermes cron create [--name NAME] [--deliver ...] [--repeat N] [--skill S] [--script PATH]
|
||||
[--no-agent] [--workdir DIR] [--model M] [--provider P] <schedule> [prompt]` — fully
|
||||
non-interactive. `hermes cron tick` = run due jobs once and exit. `hermes cron list` exists.
|
||||
|
||||
## CI pin values (heavy-tests.yml `hermes-door` job)
|
||||
- `HERMES_VERSION: "0.20.0"`
|
||||
- `HERMES_GIT_TAG: "v2026.8.3"` + `HERMES_GIT_COMMIT: "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"` —
|
||||
the installer's `--branch`/`--commit` flags pin the cloned PAYLOAD (the sha256 below only
|
||||
pins the installer script; without the tag+commit the payload would be upstream main).
|
||||
The flags are asserted, not trusted: post-install the job runs
|
||||
`git -C ~/.hermes/hermes-agent rev-parse HEAD` and loud-fails on any mismatch, so an
|
||||
installer that silently ignores unknown flags (or a moved checkout layout) can never
|
||||
run unpinned upstream code on a runner that later holds secrets.
|
||||
- `HERMES_INSTALL_SHA256: "c118ff31618dc70339049ce71061b8f1351a1c70d9c2a236ed50d8a2550c550d"`
|
||||
- Door test asserts `hermes --version` output contains `v$HERMES_VERSION` when the env var is set.
|
||||
- `hermes --version` output shape: `Hermes Agent v0.20.0 (2026.8.3)` + install dir + python lines.
|
||||
|
||||
## Multi-provider 401 gotcha (door hermeticity)
|
||||
With `model.default` pinned to `anthropic/*` but a SECOND provider key visible (env or
|
||||
.env — e.g. `OPENAI_API_KEY`), hermes's provider-auto mis-routes the request and the turn
|
||||
returns `HTTP 401: Missing Authentication header` as final text with EXIT 0. The door
|
||||
suite therefore seeds exactly ONE key (anthropic) and scrubs all provider env vars from
|
||||
hermes children (`hermesChildEnv` in test/helpers/agent-harness.ts) — the seeded
|
||||
`$HERMES_HOME/.env` is the single auth source.
|
||||
|
||||
## mcp add save-anyway (correction to an earlier note)
|
||||
A piped `Y` saves the entry EVEN when the handshake failed — the save-anyway prompt
|
||||
writes it with `enabled: false`. The success discriminators are `enabled: true` in the
|
||||
saved YAML plus `hermes mcp test <name>` exit 0 — never the add's exit code, and not the
|
||||
mere presence of the config entry.
|
||||
@@ -0,0 +1,120 @@
|
||||
# Connect GBrain to Hermes
|
||||
|
||||
> This page is the MCP-registration reference for Hermes (the NousResearch
|
||||
> `hermes-agent`). For the full brain install — CLI, engine, skills, dream
|
||||
> cycle — follow [INSTALL_FOR_AGENTS.md](../../INSTALL_FOR_AGENTS.md) first;
|
||||
> this page wires the finished brain into Hermes over stdio MCP.
|
||||
|
||||
Hermes spawns `gbrain serve` as a local stdio subprocess. No server, no tunnel,
|
||||
no token needed. Works with both PGLite and Supabase engines.
|
||||
|
||||
## Register (recommended)
|
||||
|
||||
```bash
|
||||
printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve
|
||||
```
|
||||
|
||||
`hermes mcp add` performs a real MCP handshake and tool discovery at add time,
|
||||
then prompts `Enable all N tools? [Y/n/select]:`. Three gotchas, all observed:
|
||||
|
||||
- **`--args` must be the LAST option.** Everything after it — including a
|
||||
misplaced `--env` — is swallowed into the server argv. To pass several
|
||||
environment variables, list them all after ONE `--env` flag
|
||||
(`--env A=1 B=2`); repeating the flag replaces the earlier values and the
|
||||
server is saved disabled when its handshake then fails. Put `--env` and
|
||||
`--connect-timeout` before `--command`, exactly as above.
|
||||
- **Pipe the `Y` in non-interactive contexts.** EOF on the enable-tools prompt
|
||||
prints `Cancelled.` and saves nothing. The piped `Y` saves the server with
|
||||
all tools enabled.
|
||||
- **The exit code is 0 even on connection failure or cancel.** Never assert on
|
||||
`mcp add`'s exit status — verify with `hermes mcp list` and
|
||||
`hermes mcp test gbrain` (below).
|
||||
|
||||
## Direct config (equally supported)
|
||||
|
||||
The add command writes an `mcp_servers` block into `$HERMES_HOME/config.yaml`
|
||||
(default `~/.hermes/config.yaml`). You can write it yourself instead:
|
||||
|
||||
```yaml
|
||||
mcp_servers:
|
||||
gbrain:
|
||||
command: gbrain
|
||||
args:
|
||||
- serve
|
||||
env:
|
||||
GBRAIN_HOME: /home/alice-example
|
||||
connect_timeout: 60.0
|
||||
enabled: true
|
||||
```
|
||||
|
||||
To remove gbrain, delete this block (or set `enabled: false` to disable
|
||||
without losing the config).
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
hermes mcp list # table row: gbrain ... ✓ enabled
|
||||
hermes mcp test gbrain # exits 0 and prints the discovered tool list
|
||||
```
|
||||
|
||||
Then one real round-trip:
|
||||
|
||||
```bash
|
||||
hermes -z "ask my gbrain brain: what did I import most recently?"
|
||||
```
|
||||
|
||||
`hermes -z` prints the final answer on stdout (benign notices may appear on
|
||||
stderr). Inside Hermes, gbrain's tools appear namespaced as
|
||||
`mcp_gbrain_<tool>` (e.g. `mcp_gbrain_search`).
|
||||
|
||||
## Headless auth + model pin
|
||||
|
||||
For cron jobs, CI, or any non-TTY run, Hermes needs a provider key and a
|
||||
default model configured without the interactive picker:
|
||||
|
||||
- Put the key in `$HERMES_HOME/.env`:
|
||||
|
||||
```bash
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
# or OPENROUTER_API_KEY / OPENAI_API_KEY
|
||||
```
|
||||
|
||||
- Pin the model non-interactively (`hermes model` is interactive-only — never
|
||||
use it in scripts or CI):
|
||||
|
||||
```bash
|
||||
hermes config set model.default anthropic/claude-haiku-4.5
|
||||
hermes config get model.default # reads it back
|
||||
```
|
||||
|
||||
## Pair with cron
|
||||
|
||||
Hermes cron is fully non-interactive, which makes it a natural scheduler for
|
||||
brain maintenance:
|
||||
|
||||
```bash
|
||||
hermes cron create --name gbrain-sync '0 */4 * * *' 'Run gbrain sync and report anything unusual'
|
||||
hermes cron tick # run due jobs once and exit — deterministic testing
|
||||
hermes cron list
|
||||
```
|
||||
|
||||
See [docs/guides/cron-schedule.md](../guides/cron-schedule.md) for the full
|
||||
brain maintenance protocol (sync, embed, dream cycle).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **`hermes doctor`** — global health check (installation, config, providers).
|
||||
It's not a per-server assertion; use `hermes mcp test gbrain` for that.
|
||||
- **`agent failed: No inference provider configured`** (exit 1) — Hermes has
|
||||
no model key. Set one in `$HERMES_HOME/.env` and pin `model.default` as
|
||||
above.
|
||||
- **Relocating Hermes** — both the installer and the runtime honor
|
||||
`HERMES_HOME`. All state (`config.yaml`, `.env`, `SOUL.md`, cron, logs)
|
||||
lives under it; the default is `~/.hermes`. Export it consistently or the
|
||||
gbrain registration lands in a config file the runtime never reads.
|
||||
|
||||
---
|
||||
|
||||
Documented against **Hermes Agent v0.20.0 (2026.8.3)**. Dev-facing observed-behavior
|
||||
notes (exact flag semantics, exit-code caveats, CI pin values) live in
|
||||
[HERMES-CLI-PIN.md](HERMES-CLI-PIN.md).
|
||||
@@ -0,0 +1,62 @@
|
||||
# Connect GBrain to OpenClaw
|
||||
|
||||
> This page is the MCP-registration reference card. For the full brain install
|
||||
> — CLI, engine, skills, dream cycle — follow
|
||||
> [INSTALL_FOR_AGENTS.md](../../INSTALL_FOR_AGENTS.md); the README covers the
|
||||
> bootstrap and connect paths.
|
||||
|
||||
Two supported shapes, both stdio.
|
||||
|
||||
## Option 1: ClawHub bundle plugin
|
||||
|
||||
GBrain ships [`openclaw.plugin.json`](../../openclaw.plugin.json) at the repo
|
||||
root. Installing the bundle plugin registers the MCP server for you — the
|
||||
manifest carries an `mcpServers.gbrain` entry (`./bin/gbrain serve`) plus the
|
||||
bundled skills — and declares the `gbrain-context` context engine. To route
|
||||
OpenClaw's context-engine slot through gbrain, set:
|
||||
|
||||
```
|
||||
plugins.slots.contextEngine = gbrain-context
|
||||
```
|
||||
|
||||
## Option 2: Direct `~/.openclaw/config.json`
|
||||
|
||||
The same shape gbrain's own CI uses (see the "Configure OpenClaw MCP" step in
|
||||
`.github/workflows/e2e.yml`):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "gbrain",
|
||||
"args": ["serve"],
|
||||
"env": {
|
||||
"DATABASE_URL": "postgresql://...",
|
||||
"GBRAIN_HOME": "/home/alice-example"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `env` block is optional: a PGLite brain needs no `DATABASE_URL`, and
|
||||
`GBRAIN_HOME` only matters when the brain home isn't `~/.gbrain`. Append
|
||||
`"--surface", "verbs"` to `args` for the seven-verb memory protocol
|
||||
([MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)) instead of the full
|
||||
operation catalog.
|
||||
|
||||
## Verify
|
||||
|
||||
Start an agent turn and ask it to use the brain:
|
||||
|
||||
```
|
||||
Call get_brain_identity, then search my brain for [topic].
|
||||
```
|
||||
|
||||
If the tools respond, the wiring works. `list_skills` shows everything the
|
||||
brain can do (gated by `mcp.publish_skills` on the host).
|
||||
|
||||
## Remove
|
||||
|
||||
Delete the `mcpServers.gbrain` block from `~/.openclaw/config.json`, or
|
||||
uninstall the bundle plugin.
|
||||
@@ -156,8 +156,8 @@ codex mcp add gbrain -- gbrain serve --surface verbs
|
||||
That's the whole wire-up. No token, no URL, no tunnel. The agent spawns
|
||||
`gbrain serve` as a stdio subprocess and talks to your local brain directly.
|
||||
|
||||
`--surface verbs` exposes exactly the five-verb memory protocol
|
||||
(`recall`, `remember`, `entity`, `synthesize`, `forget` —
|
||||
`--surface verbs` exposes exactly the seven-verb memory protocol
|
||||
(`recall`, `remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md), frozen + additive-forever)
|
||||
instead of the full operation catalog, so the agent sees a tight, stable surface
|
||||
instead of a 110-tool wall. Drop the flag (or pass `--surface full`) for every
|
||||
@@ -192,12 +192,12 @@ about people, companies, decisions, projects, or past context:
|
||||
tokens → `search` (cheap hybrid, no expansion). Concept, landscape, or
|
||||
"all the X that do Y" questions → `query` FIRST — it recovers synonym
|
||||
phrasings `search` misses, and a populated `search` result set is not proof
|
||||
of coverage. On the five-verb surface the same split is `recall` (retrieve)
|
||||
of coverage. On the seven-verb surface the same split is `recall` (retrieve)
|
||||
vs `synthesize` (reasoned answer). Check the brain BEFORE answering from
|
||||
memory or asking me. Never ask "who is X?" or "what did we decide about Y?"
|
||||
before checking — the brain probably already knows.
|
||||
2. **Write back.** When I make a decision, mention a new person/company, or land
|
||||
on an idea worth keeping, write it to the brain: `remember` on the five-verb
|
||||
on an idea worth keeping, write it to the brain: `remember` on the seven-verb
|
||||
surface (one fact, with provenance), or `put_page` on the full surface
|
||||
(entity pages under people/, companies/; decisions under decisions/ or
|
||||
notes/). One insight, one page, linked.
|
||||
@@ -222,7 +222,7 @@ hundreds of linked pages and patterns you didn't know were there.
|
||||
**3. Briefing from your brain (not from the internet).** *"What do I need to know
|
||||
before my 2pm with the Acme team?"* pulls your meeting history, the people,
|
||||
what's still open, what the brain doesn't know yet. The agent does your prep
|
||||
because it read your context. (`query` — `synthesize` on the five-verb surface —
|
||||
because it read your context. (`query` — `synthesize` on the seven-verb surface —
|
||||
gives you the synthesized answer with citations; this is the example on the
|
||||
[README](../../README.md).)
|
||||
|
||||
|
||||
+15
-3
@@ -661,7 +661,7 @@ four numeric segments are required first. Historical 3-segment versions
|
||||
| `CHANGELOG.md` | Top entry header `## [0.31.4.1] - YYYY-MM-DD` plus the "To take advantage of v0.31.4.1" block. | Standard Keep-a-Changelog header. |
|
||||
| `TODOS.md` | Any TODO entries that mention "follow-up from vX.Y.Z.W" use the version of the release that filed them. Update only when filing NEW follow-up TODOs. | Inline `vX.Y.Z.W` references in TODO bodies. |
|
||||
| `CLAUDE.md` | The Key Files section's per-file annotations carry `vX.Y.Z.W (#NNN)` tags noting which release introduced a behavior. Update whenever a wave's annotations get folded in. | Inline `vX.Y.Z.W (#NNN, contributed by @user)` references. |
|
||||
| `openclaw.plugin.json` | OpenClaw plugin manifest (v0.45.6.0, #4033). Hand-maintained; `test/openclaw-plugin-manifest.test.ts` fails the suite if it drifts from `package.json`. Merges from master auto-resolve it to master's version — re-bump it with the trio. | `"version": "0.45.10.0"` |
|
||||
| `openclaw.plugin.json` | OpenClaw plugin manifest (v0.45.6.0, #4033). Hand-maintained; `test/openclaw-plugin-manifest.test.ts` fails the suite if it drifts from `package.json`. Merges from master auto-resolve it to master's version — re-bump it with the trio. | `"version": "0.45.12.0"` |
|
||||
| `BOOTSTRAP_FOR_AGENTS.md` | Runbook stamp on line 1. `scripts/check-bootstrap-tag.sh` (in `bun run verify` + CI) fails when it drifts from `VERSION`; refresh it in the same commit as the bump. | `<!-- gbrain-runbook-stamp: X.Y.Z.W -->` |
|
||||
| `templates/bootstrap/template-repo/` | Vendored template tree with an embedded version stamp. Auto-derived, but NOT by `bun install`: run `bun run scripts/generate-template-repo.ts --out templates/bootstrap/template-repo` after the bump; `scripts/check-bootstrap-templates.sh` fails CI on drift. | `<!-- gbrain-template-stamp: X.Y.Z.W -->` in generated files. |
|
||||
|
||||
@@ -1233,6 +1233,16 @@ diff against gbrain's bundle when you want upstream improvements. (The legacy
|
||||
`gbrain skillpack install` managed-block model was removed in v0.33 — run
|
||||
`gbrain skillpack migrate-fence` once if upgrading from an older release.)
|
||||
|
||||
**If you are Hermes:** register gbrain as your MCP server:
|
||||
|
||||
```bash
|
||||
printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve
|
||||
```
|
||||
|
||||
Keep `--args` last (everything after it becomes server argv) and verify with
|
||||
`hermes mcp test gbrain` — the add exits 0 even on failure. Full reference:
|
||||
[docs/mcp/HERMES.md](docs/mcp/HERMES.md).
|
||||
|
||||
Whether you scaffolded or not, read `skills/RESOLVER.md` (in your workspace, or the
|
||||
bundled copy at `~/gbrain/skills/RESOLVER.md` when running from the cloned repo). It's
|
||||
the skill dispatcher — tells you which skill to read for any task. Save this to your
|
||||
@@ -1754,6 +1764,8 @@ GBrain exposes nearly all of its 100+ operations as MCP tools (stdio and HTTP; a
|
||||
- **[Claude Code](docs/mcp/CLAUDE_CODE.md)** — local: one command, `claude mcp add gbrain -- gbrain serve` (zero server, zero tunnel). Remote with just a bearer token: `gbrain connect https://your-host/mcp --token gbrain_xxx` prints a paste-ready block (or `--install` wires it up and smoke-tests the token).
|
||||
- **[Codex](docs/mcp/CODEX.md)** — `gbrain connect https://your-host/mcp --token gbrain_xxx --agent codex` (or `--install`). Codex reads the bearer from `$GBRAIN_REMOTE_TOKEN` at runtime, so the token never lands in Codex config.
|
||||
- **[Cursor / Windsurf / any stdio MCP client](docs/mcp/CLAUDE_CODE.md)** — same shape, add `{"command": "gbrain", "args": ["serve"]}` to your MCP config.
|
||||
- **[Hermes](docs/mcp/HERMES.md)** — `printf 'Y\n' | hermes mcp add gbrain --env GBRAIN_HOME=$HOME --connect-timeout 60 --command $(which gbrain) --args serve`. Keep `--args` last, and verify with `hermes mcp test gbrain` (the add exits 0 even on failure).
|
||||
- **[OpenClaw](docs/mcp/OPENCLAW.md)** — the ClawHub bundle plugin registers gbrain automatically (`openclaw.plugin.json` ships in this repo), or add `{"command": "gbrain", "args": ["serve"]}` to `~/.openclaw/config.json`'s `mcpServers`.
|
||||
- **[Claude Desktop (Cowork)](docs/mcp/CLAUDE_DESKTOP.md)** — Settings → Integrations → add the URL of your HTTP server. Remote only; the local `claude_desktop_config.json` does not work for remote servers.
|
||||
- **[Claude Cowork (team plan)](docs/mcp/CLAUDE_COWORK.md)** — org Owner adds the connector under Organization Settings → Connectors.
|
||||
- **[Perplexity Computer](docs/mcp/PERPLEXITY.md)** — `gbrain connect https://your-host/mcp --agent perplexity --oauth --register` mints a least-privilege OAuth client and prints the Issuer/Client ID/Secret to paste into Settings → Connectors (OAuth is the right path for a cloud connector; a bearer token also works for local use). Pro subscription required.
|
||||
@@ -3989,8 +4001,8 @@ gbrain serve --surface verbs # just the 7 memory verbs (quickstart surface)
|
||||
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
`--surface verbs` exposes exactly the five-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget` —
|
||||
`--surface verbs` exposes exactly the seven-verb memory protocol (`recall`,
|
||||
`remember`, `entity`, `synthesize`, `forget`, `context_pack`, `delta` —
|
||||
[MEMORY_VERBS v1](../protocol/MEMORY_VERBS_v1.md)) instead of the full catalog;
|
||||
omit the flag (default `full`) for every operation.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.45.11.0",
|
||||
"version": "0.45.12.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
|
||||
+1
-1
@@ -154,7 +154,7 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.45.11.0",
|
||||
"version": "0.45.12.0",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.5",
|
||||
|
||||
@@ -32,7 +32,12 @@ cd "$ROOT"
|
||||
BANNED_NAMES=(
|
||||
'Diana' # Diana Hu, named in CLAUDE.md privacy example
|
||||
'Wintermute' # private OpenClaw fork name (CLAUDE.md rule)
|
||||
'Hermes' # downstream agent fork name
|
||||
# 'Hermes' removed (hermes-harness wave): here it names NousResearch/hermes-agent,
|
||||
# a PUBLIC platform gbrain documents (README hero, INSTALL_FOR_AGENTS.md) and now
|
||||
# tests against (claw-test hermes runner, install door e2e).
|
||||
# test/readme-hero-anchors.test.ts REQUIRES the README to mention it. The original
|
||||
# scrub targeted conflating the public agent with PRIVATE deployment names — those
|
||||
# (Wintermute, and any future private fork names) remain banned above/below.
|
||||
'Technium' # real GP handle
|
||||
'McGrew' # ex-OpenAI exec
|
||||
'YC Labs' # internal team name
|
||||
@@ -57,9 +62,6 @@ ALLOWLIST=(
|
||||
"test/recency-decay.test.ts:Wintermute" # regression-prevention test asserting wintermute is absent (structural)
|
||||
"test/scripts/check-proposal-pii.test.ts:Wintermute" # privacy-guard test asserting docs/proposals/ rejects wintermute (structural; same meta-rule exception as check-privacy.sh)
|
||||
"test/scripts/check-proposal-pii.test.ts:WINTERMUTE" # case-insensitive sentinel literal for the same privacy-guard test
|
||||
"test/serve-stdio-lifecycle.test.ts:Hermes" # comment naming a downstream-agent scenario — pre-existing, low signal
|
||||
"test/extract.test.ts:Hermes" # markdown-link extraction test fixture — pre-existing, ambiguous (Greek god vs fork)
|
||||
"test/readme-hero-anchors.test.ts:Hermes" # v0.36.0.0 D9 anchor test — asserts README mentions Hermes as a credit
|
||||
"test/readme-hero-anchors.test.ts:OpenClaw" # v0.36.0.0 D9 anchor test — asserts README mentions OpenClaw as a credit
|
||||
# v0.36.0.0: skillpack-harvest privacy linter tests structurally
|
||||
# require the literal "Wintermute" to verify the linter catches it.
|
||||
|
||||
@@ -34,6 +34,12 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
"test/e2e/openclaw-context-engine-plugin.test.ts",
|
||||
"test/e2e/openclaw-plugin-load-real.test.ts",
|
||||
],
|
||||
// claw-test harness (command + core: runners, scenarios, seeding, friction
|
||||
// merge) feeds the scripted + shim-live E2E. The hermes door
|
||||
// (install-real-hermes.serial.test.ts) is deliberately NOT mapped — it is
|
||||
// opt-in-gated (GBRAIN_REAL_HERMES_E2E) and self-skips in run-all anyway.
|
||||
"src/commands/claw-test.ts": ["test/e2e/claw-test.test.ts"],
|
||||
"src/core/claw-test/**": ["test/e2e/claw-test.test.ts"],
|
||||
// dream.ts is a thin alias over runCycle in cycle.ts.
|
||||
"src/core/cycle.ts": ["test/e2e/cycle.test.ts", "test/e2e/dream.test.ts"],
|
||||
// Multi-source sync writes share the per-source bookmark anchor.
|
||||
|
||||
+13
-10
@@ -68,16 +68,19 @@ mkdir -p "$E2E_TMP_HOME/.gbrain"
|
||||
|
||||
# --- Hermetic env scrub: operator/agent context must not bleed into E2E ---
|
||||
# A dev shell or a Conductor workspace exports CONDUCTOR_*, MCP_*, OPENCLAW_*,
|
||||
# and GBRAIN_* config overrides (e.g. a stray GBRAIN_BRAIN_ID, GBRAIN_SOURCE,
|
||||
# GBRAIN_*_THRESHOLD, GBRAIN_SUPERVISOR_PID_FILE) that would silently change
|
||||
# test behavior — making "hermetic" E2E non-hermetic and its failures
|
||||
# unreproducible across machines. Drop them before bun starts. This is a
|
||||
# DENYLIST of operator-context prefixes (not an allowlist rebuild), so PATH,
|
||||
# HOME, TMPDIR, CI, DATABASE_URL, and bun internals survive untouched. We keep
|
||||
# GBRAIN_HOME (just set above for HOME isolation); everything else GBRAIN_* is
|
||||
# an operator override the suite must not inherit. Adapts GStack's
|
||||
# buildHermeticEnv() allowlist to gbrain's shell E2E runner.
|
||||
for _e2e_var in $(env | grep -oE '^(CONDUCTOR_|MCP_|OPENCLAW_|GBRAIN_)[A-Za-z0-9_]*' | sort -u); do
|
||||
# HERMES_*, and GBRAIN_* config overrides (e.g. a stray GBRAIN_BRAIN_ID,
|
||||
# GBRAIN_SOURCE, GBRAIN_*_THRESHOLD, GBRAIN_SUPERVISOR_PID_FILE, an operator's
|
||||
# HERMES_BIN/HERMES_HOME) that would silently change test behavior — making
|
||||
# "hermetic" E2E non-hermetic and its failures unreproducible across machines.
|
||||
# Drop them before bun starts. This is a DENYLIST of operator-context prefixes
|
||||
# (not an allowlist rebuild), so PATH, HOME, TMPDIR, CI, DATABASE_URL, and bun
|
||||
# internals survive untouched. We keep GBRAIN_HOME (just set above for HOME
|
||||
# isolation); everything else GBRAIN_* is an operator override the suite must
|
||||
# not inherit — which also scrubs GBRAIN_REAL_HERMES_E2E, so the paid hermes
|
||||
# door suite structurally cannot fire under this runner (its venue is
|
||||
# heavy-tests.yml's direct bun test). Adapts GStack's buildHermeticEnv()
|
||||
# allowlist to gbrain's shell E2E runner.
|
||||
for _e2e_var in $(env | grep -oE '^(CONDUCTOR_|MCP_|OPENCLAW_|HERMES_|GBRAIN_)[A-Za-z0-9_]*' | sort -u); do
|
||||
case "$_e2e_var" in
|
||||
GBRAIN_HOME) ;; # required for HOME isolation (set above) — keep
|
||||
*) unset "$_e2e_var" || true ;;
|
||||
|
||||
@@ -55,6 +55,7 @@ gbrain friction list # recent runs with counts
|
||||
gbrain friction render --run-id <id> # markdown report (default)
|
||||
gbrain friction render --run-id <id> --json
|
||||
gbrain friction summary --run-id <id> # friction + delight side-by-side
|
||||
gbrain friction diff --base <run-or-agent> --compare <run-or-agent> # cross-run/cross-agent comparison
|
||||
```
|
||||
|
||||
`render` defaults to `--redact` for markdown (strips `$HOME`/`$CWD` to `<HOME>`/`<CWD>` placeholders) so reports paste safely into PRs and issues.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"_AGENT_README.md": "62613f7f1e061576b6c1b18844f59bd35f2df96ca5c45c8c41fae0772b9ce4d3",
|
||||
"_brain-filing-rules.json": "cf850df6a7425464c6d63b3ace71991cc93497fa0cc8cd21acd31883e17939c6",
|
||||
"_brain-filing-rules.md": "2d2d75b7c76081c56f41b2c0a5a978c355ce957300f9b0a5575dc4079ef1f877",
|
||||
"_friction-protocol.md": "1b6e7cfa58725a6a5dc2dc787242141bc33f5fde524540d85b14ec22266140f7",
|
||||
"_friction-protocol.md": "51353207240142024ff1facc25f225712275ecdb4a034ffffdd83740c8d328e3",
|
||||
"_output-rules.md": "0722ec2ecea7f9fa2f065cf12dfe1347956a9709d29898bf9fe95e875c64b800",
|
||||
"academic-verify/SKILL.md": "1c19e27e75249d869da428ce8d060075feef8fbbfe146af58b305d11a260ebbc",
|
||||
"academic-verify/routing-eval.jsonl": "90d894a9829d9936e6ac7a6507e4de67ad26e46a1fe13b7a34e7dec1c0d887dd",
|
||||
|
||||
+608
-69
@@ -17,19 +17,22 @@
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, appendFileSync, chmodSync, cpSync, lstatSync } from 'fs';
|
||||
import { join, resolve, basename, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { tmpdir } from 'os';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { logFriction, frictionDir } from '../core/friction.ts';
|
||||
import { logFriction, frictionDir, frictionFile } from '../core/friction.ts';
|
||||
import { loadScenario, listScenarios, readBrief, type ScenarioConfig } from '../core/claw-test/scenarios.ts';
|
||||
import { parseProgressEvents, verifyExpectedPhases } from '../core/claw-test/progress-tail.ts';
|
||||
import { resolveAgentRunner, listRegisteredAgents, registerAgentRunner } from '../core/claw-test/agent-runner.ts';
|
||||
import { resolveAgentRunner, listRegisteredAgents, registerAgentRunner, validateBinPathEnv } from '../core/claw-test/agent-runner.ts';
|
||||
import { OpenClawRunner } from '../core/claw-test/runners/openclaw.ts';
|
||||
import { HermesRunner } from '../core/claw-test/runners/hermes.ts';
|
||||
import { createTranscriptSink } from '../core/claw-test/transcript-capture.ts';
|
||||
|
||||
// Ensure built-in runners are registered.
|
||||
registerAgentRunner('openclaw', () => new OpenClawRunner());
|
||||
registerAgentRunner('hermes', () => new HermesRunner());
|
||||
|
||||
interface HarnessOpts {
|
||||
scenario: string;
|
||||
@@ -38,8 +41,11 @@ interface HarnessOpts {
|
||||
keepTempdir: boolean;
|
||||
listAgents: boolean;
|
||||
help: boolean;
|
||||
/** Path to the gbrain binary used to invoke child commands. Defaults to argv[0]. */
|
||||
gbrainBin?: string;
|
||||
/** Path to the gbrain binary used to invoke child commands (always set by
|
||||
* parseArgs: GBRAIN_BIN_OVERRIDE when valid; else the compiled gbrain
|
||||
* binary, or a synthesized launcher when running under the bun runtime —
|
||||
* see resolveGbrainBin). */
|
||||
gbrainBin: string;
|
||||
}
|
||||
|
||||
interface PhaseOutcome {
|
||||
@@ -49,10 +55,24 @@ interface PhaseOutcome {
|
||||
stderrEvents: number;
|
||||
stdoutTail: string;
|
||||
stderrTail: string;
|
||||
/** Full stdout, only populated when invokeGbrain is asked to capture it. */
|
||||
stdoutFull?: string;
|
||||
}
|
||||
|
||||
const TAIL_BYTES = 4_096;
|
||||
const SUBPROCESS_TIMEOUT_MS = 5 * 60_000; // 5 minutes per phase
|
||||
/** Per-phase cap for the harness's own gbrain children (staging, scripted
|
||||
* phases, oracle probes). Env override is a test/incident escape hatch. */
|
||||
const SUBPROCESS_TIMEOUT_MS = envTimeoutMs('GBRAIN_CLAW_PHASE_TIMEOUT_MS', 5 * 60_000);
|
||||
/** Wall clock for the live agent turn — real fresh-install turns run long
|
||||
* (help text promises "5 to 10 min"), so the agent gets double the phase cap. */
|
||||
const LIVE_AGENT_TIMEOUT_MS = envTimeoutMs('GBRAIN_CLAW_AGENT_TIMEOUT_MS', 10 * 60_000);
|
||||
|
||||
function envTimeoutMs(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return fallback;
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) && n > 0 ? n : fallback;
|
||||
}
|
||||
|
||||
export async function runClawTest(args: string[]): Promise<number> {
|
||||
const opts = parseArgs(args);
|
||||
@@ -66,6 +86,18 @@ export async function runClawTest(args: string[]): Promise<number> {
|
||||
return cmdListAgents();
|
||||
}
|
||||
|
||||
// Charset guard: both values flow into filesystem paths (scenario → the
|
||||
// fixtures root join, agent → the run-id → the tempdir template), so a
|
||||
// traversal-shaped value would either escape the fixtures root or crash
|
||||
// sanitizeRunId mid-run. Usage error, exit 2.
|
||||
const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||
for (const [flag, value] of [['scenario', opts.scenario], ['agent', opts.agent]] as const) {
|
||||
if (!NAME_RE.test(value)) {
|
||||
console.error(`invalid --${flag} value ${JSON.stringify(value)}: letters, digits, dot, dash, underscore only`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
let scenario: ScenarioConfig;
|
||||
try {
|
||||
scenario = loadScenario(opts.scenario);
|
||||
@@ -83,6 +115,26 @@ export async function runClawTest(args: string[]): Promise<number> {
|
||||
console.log(`run-id: ${runId}`);
|
||||
console.log(`tempdir: ${runRoot}`);
|
||||
|
||||
// Run-start meta record. Agent-name resolution in `gbrain friction diff`
|
||||
// depends on this: a fully clean run otherwise writes zero agent-stamped
|
||||
// entries and could never be resolved by agent name. Uses the existing
|
||||
// phase-marker kind (no new FrictionKind) + additive scenario/harness_schema
|
||||
// fields.
|
||||
const agentLabel = opts.live ? opts.agent : 'scripted';
|
||||
try {
|
||||
logFriction({
|
||||
runId,
|
||||
phase: 'harness',
|
||||
kind: 'phase-marker',
|
||||
marker: 'start',
|
||||
message: `run start: scenario=${scenario.name} agent=${agentLabel}`,
|
||||
source: 'harness',
|
||||
agent: agentLabel,
|
||||
scenario: scenario.name,
|
||||
harnessSchema: 1,
|
||||
});
|
||||
} catch { /* best effort */ }
|
||||
|
||||
// SIGINT/SIGTERM finalization (D11)
|
||||
let interrupted = false;
|
||||
const onSignal = () => {
|
||||
@@ -94,7 +146,7 @@ export async function runClawTest(args: string[]): Promise<number> {
|
||||
message: 'run interrupted by signal',
|
||||
kind: 'interrupted',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
agent: agentLabel,
|
||||
});
|
||||
} catch { /* best effort */ }
|
||||
};
|
||||
@@ -108,9 +160,48 @@ export async function runClawTest(args: string[]): Promise<number> {
|
||||
} else {
|
||||
exitCode = await runScripted(opts, scenario, { runId, runRoot, gbrainHome });
|
||||
}
|
||||
} catch (e) {
|
||||
// Without this, a thrown run (spawn failure, runner detect race) would
|
||||
// reach the finally block with exitCode still 0 and stamp a
|
||||
// `run complete … exit=0` meta record — the friction log (diff/render's
|
||||
// input) silently recording success for a crashed run.
|
||||
exitCode = 1;
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error(`claw-test: run crashed: ${msg}`);
|
||||
if (e instanceof Error && e.stack) console.error(e.stack);
|
||||
try {
|
||||
logFriction({
|
||||
runId,
|
||||
phase: 'harness',
|
||||
message: `harness crashed: ${msg}`,
|
||||
severity: 'blocker',
|
||||
source: 'harness',
|
||||
agent: agentLabel,
|
||||
});
|
||||
} catch { /* best effort */ }
|
||||
} finally {
|
||||
process.off('SIGINT', onSignal);
|
||||
process.off('SIGTERM', onSignal);
|
||||
// Run-completion meta record (pairs with the start marker above).
|
||||
try {
|
||||
logFriction({
|
||||
runId,
|
||||
phase: 'harness',
|
||||
kind: 'phase-marker',
|
||||
marker: 'end',
|
||||
message: `run complete: scenario=${scenario.name} agent=${agentLabel} exit=${exitCode}`,
|
||||
source: 'harness',
|
||||
agent: agentLabel,
|
||||
scenario: scenario.name,
|
||||
});
|
||||
} catch { /* best effort */ }
|
||||
// Persist agent/child-side friction BEFORE the tempdir is deleted. The
|
||||
// children run with GBRAIN_HOME=<runRoot>, so their friction lands under
|
||||
// <runRoot>/.gbrain/friction/<runId>.jsonl — rmSync below would silently
|
||||
// destroy it on every run, leaving `friction render`/`diff` with only the
|
||||
// harness's half of the story. Merge into the parent's friction file
|
||||
// (same runId; the two sides write disjoint entries).
|
||||
mergeChildFriction(runRoot, runId);
|
||||
if (!opts.keepTempdir && !interrupted) {
|
||||
try { rmSync(runRoot, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
} else {
|
||||
@@ -148,6 +239,36 @@ export async function runClawTest(args: string[]): Promise<number> {
|
||||
*/
|
||||
const POSTGRES_POLLUTION_ENV_VARS = ['DATABASE_URL', 'GBRAIN_DATABASE_URL'];
|
||||
|
||||
/**
|
||||
* Child env for gbrain invocations (scripted phases AND live-mode staging /
|
||||
* oracle probes): parent env minus Postgres-pointing vars AND minus every
|
||||
* other GBRAIN_* routing/tuning var — a stray operator GBRAIN_BRAIN_ID /
|
||||
* GBRAIN_SOURCE / threshold override would misroute the staging and oracle
|
||||
* probes and produce false verify verdicts (the same class scripts/run-e2e.sh
|
||||
* scrubs for e2e hermeticity). The two vars the harness owns are re-applied
|
||||
* last so a parent override can't win.
|
||||
*/
|
||||
function buildChildEnv(ctx: { runId: string; gbrainHome: string }): Record<string, string> {
|
||||
const parentEnv = process.env as Record<string, string | undefined>;
|
||||
const childEnv: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(parentEnv)) {
|
||||
if (v === undefined) continue;
|
||||
if (POSTGRES_POLLUTION_ENV_VARS.includes(k)) continue;
|
||||
if (k.startsWith('GBRAIN_')) continue;
|
||||
childEnv[k] = v;
|
||||
}
|
||||
childEnv.GBRAIN_HOME = ctx.gbrainHome;
|
||||
childEnv.GBRAIN_FRICTION_RUN_ID = ctx.runId;
|
||||
return childEnv;
|
||||
}
|
||||
|
||||
/** The hermetic run's PGLite path (configDir appends '.gbrain'). One spelling
|
||||
* for all probe/seed sites — a drifted copy would silently probe a
|
||||
* nonexistent db and fail the upgrade oracle as "unreadable". */
|
||||
function pgliteDbPath(gbrainHome: string): string {
|
||||
return join(gbrainHome, '.gbrain', 'brain.pglite');
|
||||
}
|
||||
|
||||
async function runScripted(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
@@ -157,17 +278,7 @@ async function runScripted(
|
||||
// The harness is PGLite-only by design; an inherited DATABASE_URL
|
||||
// would force loadConfig() to flip the engine to 'postgres' at the
|
||||
// next phase boundary and break the hermetic-tempdir contract.
|
||||
const parentEnv = process.env as Record<string, string | undefined>;
|
||||
const childEnv: Record<string, string> = { GBRAIN_HOME: ctx.gbrainHome, GBRAIN_FRICTION_RUN_ID: ctx.runId };
|
||||
for (const [k, v] of Object.entries(parentEnv)) {
|
||||
if (v === undefined) continue;
|
||||
if (POSTGRES_POLLUTION_ENV_VARS.includes(k)) continue;
|
||||
childEnv[k] = v;
|
||||
}
|
||||
// Re-apply the explicit overrides so a parent GBRAIN_HOME / GBRAIN_FRICTION_RUN_ID
|
||||
// can't accidentally win the merge.
|
||||
childEnv.GBRAIN_HOME = ctx.gbrainHome;
|
||||
childEnv.GBRAIN_FRICTION_RUN_ID = ctx.runId;
|
||||
const childEnv = buildChildEnv(ctx);
|
||||
|
||||
const phases: { name: string; argv: string[] }[] = [];
|
||||
// Phase 2: install_brain. `--no-embedding` defers embedding setup so the
|
||||
@@ -200,35 +311,51 @@ async function runScripted(
|
||||
// Phase 6: verify
|
||||
phases.push({ name: 'verify', argv: ['doctor', '--json', '--progress-json'] });
|
||||
|
||||
// Pre-phase: upgrade scenario seeds the database
|
||||
if (scenario.kind === 'upgrade' && scenario.seedRelative) {
|
||||
const seedSql = join(scenario.dir, scenario.seedRelative, 'dump.sql');
|
||||
if (existsSync(seedSql)) {
|
||||
const dbPath = join(ctx.gbrainHome, '.gbrain', 'brain.pglite');
|
||||
mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
|
||||
const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
|
||||
try {
|
||||
await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
|
||||
console.log(`[seed] replayed ${seedSql} → ${dbPath}`);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'seed',
|
||||
message: `seed replay failed: ${msg}`,
|
||||
severity: 'blocker',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return 1;
|
||||
}
|
||||
// Pre-phase: upgrade scenario seeds the database. A missing dump is a LOUD
|
||||
// failure, not a skip: skipping would init a current-version database and
|
||||
// report a false-green "upgrade" that never exercised a migration.
|
||||
if (scenario.kind === 'upgrade') {
|
||||
const seedSql = scenario.seedRelative ? join(scenario.dir, scenario.seedRelative, 'dump.sql') : null;
|
||||
if (!seedSql || !existsSync(seedSql)) {
|
||||
const msg = seedSql
|
||||
? `upgrade scenario has no seed dump at ${seedSql}`
|
||||
: 'upgrade scenario declares no seed dir';
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'seed',
|
||||
message: msg,
|
||||
severity: 'blocker',
|
||||
hint: 'upgrade runs need a real dump.sql to measure the migration (see the TODOS entry for the v0.18 seed dump)',
|
||||
source: 'harness',
|
||||
agent: 'scripted',
|
||||
});
|
||||
console.error(`[seed] ${msg}`);
|
||||
return 1;
|
||||
}
|
||||
const dbPath = pgliteDbPath(ctx.gbrainHome);
|
||||
mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
|
||||
const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
|
||||
try {
|
||||
await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
|
||||
console.log(`[seed] replayed ${seedSql} → ${dbPath}`);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'seed',
|
||||
message: `seed replay failed: ${msg}`,
|
||||
severity: 'blocker',
|
||||
source: 'harness',
|
||||
agent: 'scripted',
|
||||
});
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const allStderr: string[] = [];
|
||||
const outcomes: PhaseOutcome[] = [];
|
||||
for (const phase of phases) {
|
||||
const outcome = await invokeGbrain(opts.gbrainBin ?? 'gbrain', phase.argv, ctx.runRoot, childEnv);
|
||||
const outcome = await invokeGbrain(opts.gbrainBin, phase.argv, ctx.runRoot, childEnv);
|
||||
outcome.phase = phase.name;
|
||||
outcomes.push(outcome);
|
||||
allStderr.push(outcome.stderrTail);
|
||||
@@ -240,7 +367,7 @@ async function runScripted(
|
||||
severity: 'error',
|
||||
hint: outcome.stderrTail.trim().slice(0, 500),
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
agent: 'scripted',
|
||||
});
|
||||
return 1;
|
||||
} else {
|
||||
@@ -251,7 +378,7 @@ async function runScripted(
|
||||
kind: 'phase-marker',
|
||||
marker: 'end',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
agent: 'scripted',
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -268,7 +395,7 @@ async function runScripted(
|
||||
severity: 'blocker',
|
||||
hint: 'either the command did not run or it did not emit progress events; check phase log above',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
agent: 'scripted',
|
||||
});
|
||||
}
|
||||
return 1;
|
||||
@@ -281,6 +408,19 @@ async function runScripted(
|
||||
// Live mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Per-agent install hints for the agent_detect blocker. */
|
||||
const AGENT_INSTALL_HINTS: Record<string, string> = {
|
||||
openclaw: 'install openclaw or set OPENCLAW_BIN',
|
||||
hermes: 'install hermes (https://hermes-agent.nousresearch.com) or set HERMES_BIN',
|
||||
};
|
||||
|
||||
/**
|
||||
* Live mode. Hermeticity posture (deliberate, documented): live mode runs the
|
||||
* OPERATOR's configured agent — the real agent home (~/.openclaw, ~/.hermes,
|
||||
* model settings, skills) is inherited — against a HERMETIC BRAIN
|
||||
* (GBRAIN_HOME=tempdir). The fully hermetic lane is the door e2e
|
||||
* (install-real-hermes.serial.test.ts), which isolates the agent home too.
|
||||
*/
|
||||
async function runLive(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
@@ -302,17 +442,49 @@ async function runLive(
|
||||
phase: 'agent_detect',
|
||||
message: `agent ${opts.agent} not available: ${detected.reason ?? 'unknown'}`,
|
||||
severity: 'blocker',
|
||||
hint: opts.agent === 'openclaw' ? 'install openclaw or set OPENCLAW_BIN' : undefined,
|
||||
hint: AGENT_INSTALL_HINTS[opts.agent],
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return 2;
|
||||
}
|
||||
|
||||
// ---- Stage the scenario (scenario-driven; mirrors the scripted branch) ----
|
||||
// The BRIEF's preconditions must actually exist before the agent reads it:
|
||||
// fresh-install promises "workspace already has an AGENTS.md", "3 small
|
||||
// markdown pages already there" (./brain), and "user just ran gbrain init".
|
||||
// Without staging, every live run starts in an empty tempdir and the run
|
||||
// measures recovery-from-broken-fixture, not gbrain friction.
|
||||
const childEnv = buildChildEnv(ctx);
|
||||
const gbrainBin = opts.gbrainBin;
|
||||
const stageFailed = await stageLiveScenario(opts, scenario, ctx, childEnv, gbrainBin);
|
||||
if (stageFailed !== 0) return stageFailed;
|
||||
|
||||
// Upgrade oracle needs the pre-turn schema version (non-mutating probe —
|
||||
// any gbrain CLI connect would auto-apply migrations and do the agent's
|
||||
// work for it).
|
||||
const dbPath = pgliteDbPath(ctx.gbrainHome);
|
||||
let preVersion: number | null = null;
|
||||
if (scenario.kind === 'upgrade') {
|
||||
const { readPgliteSchemaVersion } = await import('../core/claw-test/seed-pglite.ts');
|
||||
preVersion = await readPgliteSchemaVersion(dbPath);
|
||||
}
|
||||
|
||||
// ---- PATH shim: the BRIEF says `gbrain …`; make bare `gbrain` resolve to
|
||||
// THIS harness's binary (operator PATH may have none, or a stale global). ----
|
||||
const shimDir = join(ctx.runRoot, '.harness-bin');
|
||||
mkdirSync(shimDir, { recursive: true });
|
||||
const shimPath = join(shimDir, 'gbrain');
|
||||
// Single-quoted: validateBinPathEnv rejects quote/metacharacter values, so
|
||||
// the interpolation cannot break out of the quoting.
|
||||
writeFileSync(shimPath, `#!/bin/sh\nexec '${gbrainBin}' "$@"\n`, 'utf-8');
|
||||
chmodSync(shimPath, 0o755);
|
||||
|
||||
const sink = createTranscriptSink(ctx.transcriptPath);
|
||||
const env: Record<string, string> = {
|
||||
GBRAIN_HOME: ctx.gbrainHome,
|
||||
GBRAIN_FRICTION_RUN_ID: ctx.runId,
|
||||
PATH: `${shimDir}:${process.env.PATH ?? ''}`,
|
||||
};
|
||||
|
||||
const brief = readBrief(scenario);
|
||||
@@ -322,7 +494,7 @@ async function runLive(
|
||||
cwd: ctx.runRoot,
|
||||
brief,
|
||||
env,
|
||||
timeoutMs: SUBPROCESS_TIMEOUT_MS,
|
||||
timeoutMs: LIVE_AGENT_TIMEOUT_MS,
|
||||
transcriptSink: sink,
|
||||
});
|
||||
} finally {
|
||||
@@ -340,9 +512,270 @@ async function runLive(
|
||||
});
|
||||
return result.exitCode;
|
||||
}
|
||||
|
||||
// ---- Success oracle: exit code alone passes an agent that did nothing. ----
|
||||
return verifyLiveOutcome(opts, scenario, ctx, childEnv, gbrainBin, preVersion);
|
||||
}
|
||||
|
||||
/** Stage the workspace per scenario.kind before the agent turn. Returns 0 or a failing exit code. */
|
||||
async function stageLiveScenario(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
ctx: { runId: string; runRoot: string; gbrainHome: string },
|
||||
childEnv: Record<string, string>,
|
||||
gbrainBin: string,
|
||||
): Promise<number> {
|
||||
const failStage = (message: string, hint?: string): number => {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'stage',
|
||||
message,
|
||||
severity: 'blocker',
|
||||
hint,
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
console.error(`[stage] ${message}`);
|
||||
return 1;
|
||||
};
|
||||
|
||||
if (scenario.kind === 'upgrade') {
|
||||
// Seed ONLY — running init here would walk the migration chain forward
|
||||
// and do the very upgrade the agent turn is supposed to perform (any
|
||||
// gbrain connect auto-migrates). Same seed-first order as scripted mode.
|
||||
if (scenario.seedRelative) {
|
||||
const seedSql = join(scenario.dir, scenario.seedRelative, 'dump.sql');
|
||||
if (existsSync(seedSql)) {
|
||||
const dbPath = pgliteDbPath(ctx.gbrainHome);
|
||||
mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
|
||||
const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
|
||||
try {
|
||||
await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
|
||||
console.log(`[stage] replayed ${seedSql} → ${dbPath}`);
|
||||
} catch (e) {
|
||||
return failStage(`seed replay failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
} else {
|
||||
return failStage(`upgrade scenario has no seed dump at ${seedSql}`, 'upgrade runs need a real dump.sql to measure the migration (see the TODOS entry for the v0.18 seed dump)');
|
||||
}
|
||||
} else {
|
||||
return failStage('upgrade scenario declares no seed dir');
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// fresh-install: copy the scenario's brain pages + an AGENTS.md stub, then
|
||||
// init the brain (the BRIEF says the user "just ran gbrain init").
|
||||
if (scenario.brainRelative) {
|
||||
const src = join(scenario.dir, scenario.brainRelative);
|
||||
if (!existsSync(src)) {
|
||||
// Fail loudly, matching the upgrade branch's missing-seed blocker: a
|
||||
// silent skip here would fail the query oracle later with the
|
||||
// misleading hint "the agent likely skipped the import step" when the
|
||||
// real cause is a broken fixture.
|
||||
return failStage(`fresh-install scenario declares brain dir ${scenario.brainRelative} but it does not exist at ${src}`);
|
||||
}
|
||||
cpSync(src, join(ctx.runRoot, 'brain'), { recursive: true });
|
||||
}
|
||||
const agentsMd = join(ctx.runRoot, 'AGENTS.md');
|
||||
if (!existsSync(agentsMd)) {
|
||||
// Deliberately references NO skill files: staging creates none, and a row
|
||||
// pointing at a missing SKILL.md flips doctor's resolver_health to fail
|
||||
// (caught in rehearsal). Post-v0.33 scaffolded skills route via their own
|
||||
// frontmatter triggers, so a prose stub satisfies the BRIEF's
|
||||
// "workspace already has an AGENTS.md routing file" precondition.
|
||||
writeFileSync(
|
||||
agentsMd,
|
||||
'# Workspace routing\n\nSkills scaffolded under `skills/` route via their frontmatter `triggers:`.\n',
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
const init = await invokeGbrain(gbrainBin, ['init', '--pglite', '--no-embedding'], ctx.runRoot, childEnv);
|
||||
if (init.exitCode !== 0) {
|
||||
return failStage(`gbrain init failed during staging (exit ${init.exitCode})`, init.stderrTail.trim().slice(0, 500));
|
||||
}
|
||||
console.log('[stage] fresh-install workspace staged (brain pages + AGENTS.md + init)');
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Post-turn verification. Logs friction (phase 'verify') and returns 1 on any failure. */
|
||||
async function verifyLiveOutcome(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
ctx: { runId: string; runRoot: string; gbrainHome: string },
|
||||
childEnv: Record<string, string>,
|
||||
gbrainBin: string,
|
||||
preVersion: number | null,
|
||||
): Promise<number> {
|
||||
const failures: { message: string; hint?: string }[] = [];
|
||||
|
||||
if (scenario.kind === 'upgrade') {
|
||||
// The upgrade oracle is the schema version reaching LATEST during the
|
||||
// agent turn, read via the non-mutating direct-PGLite probe (doctor/any
|
||||
// CLI connect would apply the migrations itself and mask a do-nothing
|
||||
// agent). MUST run before any declared query oracle below — the query's
|
||||
// own CLI connect migrates, which would corrupt a later version read.
|
||||
const { readPgliteSchemaVersion } = await import('../core/claw-test/seed-pglite.ts');
|
||||
const { LATEST_VERSION } = await import('../core/migrate.ts');
|
||||
const dbPath = pgliteDbPath(ctx.gbrainHome);
|
||||
const postVersion = await readPgliteSchemaVersion(dbPath);
|
||||
if (preVersion === null || postVersion === null) {
|
||||
failures.push({ message: `upgrade oracle: schema version unreadable (pre=${preVersion} post=${postVersion})` });
|
||||
} else if (postVersion <= preVersion) {
|
||||
failures.push({
|
||||
message: `upgrade oracle: schema version did not advance during the agent turn (pre=${preVersion} post=${postVersion})`,
|
||||
hint: 'the agent never ran a gbrain command that walks the migration chain',
|
||||
});
|
||||
} else if (postVersion < LATEST_VERSION) {
|
||||
// Advancing one step is not an upgrade: any gbrain connect migrates to
|
||||
// latest, so a partial version means the agent's run died mid-chain.
|
||||
failures.push({
|
||||
message: `upgrade oracle: schema version advanced but stopped short of latest (pre=${preVersion} post=${postVersion} latest=${LATEST_VERSION})`,
|
||||
hint: 'a gbrain command started the migration chain but did not complete it',
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// doctor: fresh --no-embedding brains report status "warnings" (observed
|
||||
// — embedding setup is deferred by design), so healthy AND warnings both
|
||||
// pass. Anything else — including output the harness cannot parse — is a
|
||||
// failure: an oracle that shrugs at unparsable output fails open.
|
||||
const doc = await invokeGbrain(gbrainBin, ['doctor', '--json'], ctx.runRoot, childEnv, { captureFullStdout: true });
|
||||
if (doc.exitCode !== 0) {
|
||||
failures.push({ message: `verify: doctor exited ${doc.exitCode}`, hint: doc.stderrTail.trim().slice(0, 300) });
|
||||
} else {
|
||||
const report = parseLastJson(doc.stdoutFull ?? doc.stdoutTail);
|
||||
const status = report && typeof report === 'object' ? (report as Record<string, unknown>).status : undefined;
|
||||
if (report === null || typeof status !== 'string') {
|
||||
failures.push({ message: 'verify: doctor exited 0 but its JSON output was unparsable' });
|
||||
} else if (status !== 'healthy' && status !== 'warnings') {
|
||||
failures.push({ message: `verify: doctor reports status ${JSON.stringify(status)}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A declared oracle is enforced for EVERY kind (loadScenario validates it
|
||||
// for every kind — accepting config it never enforces would be a silent
|
||||
// no-op; an upgrade agent that migrates the schema but loses the seeded
|
||||
// data must still fail a declared query oracle).
|
||||
const oracle = scenario.oracle;
|
||||
if (oracle?.query) {
|
||||
const q = await invokeGbrain(gbrainBin, ['query', oracle.query, '--json'], ctx.runRoot, childEnv, { captureFullStdout: true });
|
||||
const parsed = q.exitCode === 0 ? parseLastJson(q.stdoutFull ?? q.stdoutTail) : null;
|
||||
const min = oracle.minResults ?? 1;
|
||||
// query --json emits a bare array; anything else on a zero exit means the
|
||||
// command's contract broke — fail even when min_results is 0, because
|
||||
// "0 results required" never licenses unparsable output.
|
||||
if (q.exitCode === 0 && !Array.isArray(parsed)) {
|
||||
failures.push({
|
||||
message: `verify: query ${JSON.stringify(oracle.query)} exited 0 but its JSON output was unparsable`,
|
||||
});
|
||||
} else {
|
||||
const count = Array.isArray(parsed) ? parsed.length : 0;
|
||||
if (q.exitCode !== 0 || count < min) {
|
||||
failures.push({
|
||||
message: `verify: query ${JSON.stringify(oracle.query)} returned ${count} result(s), expected >= ${min} (exit ${q.exitCode})`,
|
||||
hint: 'the agent likely skipped the import step from the brief',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const rel of oracle?.filesExist ?? []) {
|
||||
if (!existsSync(join(ctx.runRoot, rel))) {
|
||||
failures.push({
|
||||
message: `verify: expected file missing after run: ${rel}`,
|
||||
hint: 'the agent likely skipped a brief step that produces this file',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const f of failures) {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'verify',
|
||||
message: f.message,
|
||||
severity: 'error',
|
||||
hint: f.hint,
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
console.error(`[verify] ${f.message}`);
|
||||
}
|
||||
return failures.length ? 1 : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the trailing JSON document from CLI stdout (defensive: banners or
|
||||
* notices may precede the payload).
|
||||
*/
|
||||
function parseLastJson(stdout: string): unknown {
|
||||
const text = stdout.trim();
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch { /* fall through */ }
|
||||
const starts = ['{', '['];
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (starts.includes(text[i])) {
|
||||
try {
|
||||
return JSON.parse(text.slice(i));
|
||||
} catch { /* keep scanning */ }
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Refuse to import child friction files larger than this — the file lives in
|
||||
* a workspace the AGENT writes to, so its size is untrusted. */
|
||||
const CHILD_FRICTION_MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* E0: merge the child-side friction file (written under the run's hermetic
|
||||
* GBRAIN_HOME) into the parent process's friction dir so it survives tempdir
|
||||
* cleanup. Best-effort — a merge failure never fails the run.
|
||||
*
|
||||
* The child file is UNTRUSTED input (in live mode the agent can write
|
||||
* arbitrary bytes at that path), and the destination is the operator's
|
||||
* permanent friction log: require a regular file (no symlink — an agent-
|
||||
* dropped link could import any readable file on the box), cap the size, and
|
||||
* append only lines that parse as JSON objects so the log stays valid JSONL.
|
||||
* Exported for tests.
|
||||
*/
|
||||
export function mergeChildFriction(runRoot: string, runId: string): void {
|
||||
try {
|
||||
const childFile = join(runRoot, '.gbrain', 'friction', `${runId}.jsonl`);
|
||||
const st = lstatSync(childFile, { throwIfNoEntry: false });
|
||||
if (!st) return;
|
||||
if (!st.isFile()) {
|
||||
console.error(`[friction] skipping child friction merge: ${childFile} is not a regular file`);
|
||||
return;
|
||||
}
|
||||
if (st.size > CHILD_FRICTION_MAX_BYTES) {
|
||||
console.error(`[friction] skipping child friction merge: ${childFile} is ${st.size} bytes (cap ${CHILD_FRICTION_MAX_BYTES})`);
|
||||
return;
|
||||
}
|
||||
const parentFile = frictionFile(runId);
|
||||
if (resolve(childFile) === resolve(parentFile)) return;
|
||||
const raw = readFileSync(childFile, 'utf-8');
|
||||
if (!raw.trim()) return;
|
||||
const kept: string[] = [];
|
||||
let skipped = 0;
|
||||
for (const line of raw.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) kept.push(line);
|
||||
else skipped++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
if (skipped) console.error(`[friction] child friction merge skipped ${skipped} non-JSONL line(s)`);
|
||||
if (!kept.length) return;
|
||||
const dir = frictionDir();
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
appendFileSync(parentFile, kept.join('\n') + '\n', 'utf-8');
|
||||
} catch { /* best effort */ }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subprocess helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -352,17 +785,60 @@ function invokeGbrain(
|
||||
argv: string[],
|
||||
cwd: string,
|
||||
env: Record<string, string>,
|
||||
invokeOpts?: { captureFullStdout?: boolean },
|
||||
): Promise<PhaseOutcome> {
|
||||
return new Promise((resolve) => {
|
||||
return new Promise((resolvePromise) => {
|
||||
const start = Date.now();
|
||||
const child = spawn(bin, argv, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], shell: false });
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
child.stdout?.on('data', (b: Buffer) => stdout.push(b));
|
||||
child.stderr?.on('data', (b: Buffer) => stderr.push(b));
|
||||
|
||||
// A hung child (e.g. a leaked PGLite lock holder from the agent turn)
|
||||
// must not wedge the harness/CI job forever: SIGTERM at the phase cap,
|
||||
// SIGKILL if it lingers.
|
||||
let timedOut = false;
|
||||
let killTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const wallClockTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
try { child.kill('SIGTERM'); } catch { /* already gone */ }
|
||||
killTimer = setTimeout(() => {
|
||||
try { child.kill('SIGKILL'); } catch { /* already gone */ }
|
||||
}, 10_000);
|
||||
}, SUBPROCESS_TIMEOUT_MS);
|
||||
const clearTimers = () => {
|
||||
clearTimeout(wallClockTimer);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
};
|
||||
|
||||
let settled = false;
|
||||
const settle = (code: number | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimers();
|
||||
let stderrText = Buffer.concat(stderr).toString('utf-8');
|
||||
if (timedOut) stderrText += `\nharness: killed after ${SUBPROCESS_TIMEOUT_MS}ms phase timeout`;
|
||||
const stdoutText = Buffer.concat(stdout).toString('utf-8');
|
||||
resolvePromise({
|
||||
phase: '',
|
||||
exitCode: typeof code === 'number' ? code : (timedOut ? 124 : 1),
|
||||
durationMs: Date.now() - start,
|
||||
stderrEvents: parseProgressEvents(stderrText).length,
|
||||
stdoutTail: tailOf(stdoutText),
|
||||
stderrTail: stderrText,
|
||||
// The 4KB tail is fine for logging but NOT for parsing JSON payloads
|
||||
// (doctor --json exceeds it and would lose its opening brace).
|
||||
...(invokeOpts?.captureFullStdout ? { stdoutFull: stdoutText } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
child.on('error', (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimers();
|
||||
const stderrJoined = Buffer.concat(stderr).toString('utf-8') + '\nspawn error: ' + err.message;
|
||||
resolve({
|
||||
resolvePromise({
|
||||
phase: '',
|
||||
exitCode: 127,
|
||||
durationMs: Date.now() - start,
|
||||
@@ -371,16 +847,13 @@ function invokeGbrain(
|
||||
stderrTail: tailOf(stderrJoined),
|
||||
});
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
const stderrText = Buffer.concat(stderr).toString('utf-8');
|
||||
resolve({
|
||||
phase: '',
|
||||
exitCode: typeof code === 'number' ? code : 1,
|
||||
durationMs: Date.now() - start,
|
||||
stderrEvents: parseProgressEvents(stderrText).length,
|
||||
stdoutTail: tailOf(Buffer.concat(stdout).toString('utf-8')),
|
||||
stderrTail: stderrText,
|
||||
});
|
||||
// 'close' (pipes drained) is the clean path; 'exit' + grace covers a
|
||||
// grandchild that inherits the pipes and outlives the kill — without it a
|
||||
// timed-out phase whose child leaked a subprocess would wedge forever.
|
||||
child.on('close', (code) => settle(code));
|
||||
child.on('exit', (code) => {
|
||||
const t = setTimeout(() => settle(code), 2_000);
|
||||
t.unref?.();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -402,7 +875,7 @@ function parseArgs(args: string[]): HarnessOpts {
|
||||
keepTempdir: false,
|
||||
listAgents: false,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
gbrainBin: process.env.GBRAIN_BIN_OVERRIDE || process.execPath,
|
||||
gbrainBin: resolveGbrainBin(),
|
||||
};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
@@ -415,6 +888,60 @@ function parseArgs(args: string[]): HarnessOpts {
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the gbrain binary child invocations use. GBRAIN_BIN_OVERRIDE goes
|
||||
* through the same absolute/no-dotdot/no-metacharacter validation as the
|
||||
* *_BIN runner overrides — the value is interpolated into a generated
|
||||
* PATH-shim script in live mode, so a relative value would re-resolve through
|
||||
* the shimmed PATH and self-exec forever, and quoting-hostile characters
|
||||
* would become code. Invalid overrides are rejected loudly (stderr) and the
|
||||
* harness falls back to the current executable.
|
||||
*
|
||||
* The fallback is NOT bare process.execPath: under `bun run src/cli.ts` (the
|
||||
* canonical source install) or a bun-global launcher, execPath is the Bun
|
||||
* RUNTIME, and children would run `bun init` / `bun import` instead of gbrain
|
||||
* — scaffolding a Bun project in the hermetic workspace and failing the rest
|
||||
* of the run. When execPath looks like bun, synthesize a launcher shim that
|
||||
* re-enters this checkout's cli.ts; only a compiled gbrain binary returns
|
||||
* execPath directly.
|
||||
*/
|
||||
let cachedGbrainBin: string | null = null;
|
||||
|
||||
function resolveGbrainBin(): string {
|
||||
if (cachedGbrainBin) return cachedGbrainBin;
|
||||
cachedGbrainBin = resolveGbrainBinUncached();
|
||||
return cachedGbrainBin;
|
||||
}
|
||||
|
||||
function resolveGbrainBinUncached(): string {
|
||||
const override = process.env.GBRAIN_BIN_OVERRIDE?.trim();
|
||||
if (override) {
|
||||
const invalid = validateBinPathEnv('GBRAIN_BIN_OVERRIDE', override);
|
||||
if (!invalid) return override;
|
||||
console.error(`ignoring ${invalid}; falling back to the current executable`);
|
||||
}
|
||||
const exe = process.execPath;
|
||||
if (/^bun(-profile)?(\.exe)?$/i.test(basename(exe))) {
|
||||
// src/commands/claw-test.ts → ../cli.ts. Under a compiled binary this
|
||||
// branch never fires (execPath is the gbrain binary itself); under bun
|
||||
// (dev checkout or bun-global install) import.meta resolves to the real
|
||||
// source file next to cli.ts.
|
||||
const cliTs = resolve(dirname(fileURLToPath(import.meta.url)), '..', 'cli.ts');
|
||||
if (existsSync(cliTs) && !/['\n\r]/.test(exe) && !/['\n\r]/.test(cliTs)) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gbrain-launcher-'));
|
||||
const launcher = join(dir, 'gbrain');
|
||||
writeFileSync(launcher, `#!/bin/sh\nexec '${exe}' '${cliTs}' "$@"\n`, 'utf-8');
|
||||
chmodSync(launcher, 0o755);
|
||||
return launcher;
|
||||
}
|
||||
console.error(
|
||||
'claw-test: running under the bun runtime but the gbrain CLI entrypoint could not be located — ' +
|
||||
'child gbrain invocations would run bun itself. Set GBRAIN_BIN_OVERRIDE to a gbrain binary.',
|
||||
);
|
||||
}
|
||||
return exe;
|
||||
}
|
||||
|
||||
function newRunId(agent: string): string {
|
||||
const now = new Date();
|
||||
const ts = now.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-');
|
||||
@@ -422,23 +949,30 @@ function newRunId(agent: string): string {
|
||||
return `claw-test-${ts}-${agent}-${suf}`;
|
||||
}
|
||||
|
||||
function cmdListAgents(): number {
|
||||
async function cmdListAgents(): Promise<number> {
|
||||
const names = listRegisteredAgents();
|
||||
if (!names.length) {
|
||||
console.log('no agents registered');
|
||||
return 0;
|
||||
}
|
||||
for (const name of names) {
|
||||
// Detect concurrently but AWAIT all of them, then print in
|
||||
// listRegisteredAgents() order (the accessor sorts alphabetically).
|
||||
// The prior fire-and-forget .then() version returned before any detection
|
||||
// resolved, so output could vanish in CLI teardown.
|
||||
const lines = await Promise.all(names.map(async (name) => {
|
||||
try {
|
||||
const runner = resolveAgentRunner(name);
|
||||
runner.detect().then((d) => {
|
||||
const status = d.available ? `available at ${d.binPath}` : `unavailable: ${d.reason}`;
|
||||
console.log(`${name}: ${status}`);
|
||||
}).catch(() => { /* best effort */ });
|
||||
try {
|
||||
const d = await runner.detect();
|
||||
return `${name}: ${d.available ? `available at ${d.binPath}` : `unavailable: ${d.reason}`}`;
|
||||
} catch {
|
||||
return `${name}: (detect error)`;
|
||||
}
|
||||
} catch {
|
||||
console.log(`${name}: (factory error)`);
|
||||
return `${name}: (factory error)`;
|
||||
}
|
||||
}
|
||||
}));
|
||||
for (const line of lines) console.log(line);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -456,8 +990,13 @@ Defaults:
|
||||
Scripted mode runs canonical commands without an LLM (CI gate).
|
||||
Live mode spawns a real agent and lets it drive (~5–10 min, costs tokens).
|
||||
|
||||
Live mode runs YOUR configured agent (it may read/write your real agent home,
|
||||
e.g. ~/.openclaw or ~/.hermes) against a hermetic brain. The door e2e suite is
|
||||
the fully hermetic lane.
|
||||
|
||||
Examples:
|
||||
gbrain claw-test --scenario fresh-install
|
||||
gbrain claw-test --scenario upgrade-from-v0.18 --keep-tempdir
|
||||
gbrain claw-test --live --agent openclaw`);
|
||||
gbrain claw-test --live --agent openclaw
|
||||
gbrain claw-test --live --agent hermes`);
|
||||
}
|
||||
|
||||
+324
-5
@@ -1,21 +1,24 @@
|
||||
/**
|
||||
* gbrain friction — friction reporter CLI.
|
||||
*
|
||||
* Four subcommands in v1 (analytical/clustering ones move to v1.1):
|
||||
* Five subcommands (remaining analytical ones — trend, migration-stub — stay v1.1):
|
||||
* gbrain friction log Append a friction or delight entry
|
||||
* gbrain friction render Render a run as markdown or JSON
|
||||
* gbrain friction list List recent runs with counts
|
||||
* gbrain friction summary Side-by-side friction + delight summary
|
||||
* gbrain friction diff Compare two runs (or agents): unique-to-each + shared-but-changed
|
||||
*
|
||||
* Subcommands stay thin (≤ ~30 LOC each). Core logic lives in src/core/friction.ts.
|
||||
* Subcommands stay thin (≤ ~30 LOC each). Reader/writer/redaction logic lives
|
||||
* in src/core/friction.ts; the diff computation lives here (it is CLI-only).
|
||||
*
|
||||
* The CLI is dispatched from src/cli.ts. See `gbrain friction --help`.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import {
|
||||
logFriction, readFriction, listRuns, renderReport, renderSummary,
|
||||
activeRunId, frictionFile,
|
||||
type FrictionKind, type FrictionSeverity,
|
||||
activeRunId, frictionFile, frictionDir, redactEntry,
|
||||
type FrictionKind, type FrictionSeverity, type FrictionEntry, type ReadResult,
|
||||
} from '../core/friction.ts';
|
||||
|
||||
const VALID_KINDS = new Set<FrictionKind>(['friction', 'delight', 'phase-marker', 'interrupted']);
|
||||
@@ -28,6 +31,7 @@ export function runFriction(args: string[]): number {
|
||||
case 'render': return cmdRender(rest);
|
||||
case 'list': return cmdList(rest);
|
||||
case 'summary': return cmdSummary(rest);
|
||||
case 'diff': return cmdDiff(rest);
|
||||
case undefined:
|
||||
case '--help':
|
||||
case '-h':
|
||||
@@ -150,6 +154,317 @@ function cmdSummary(args: string[]): number {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// diff
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Entry identity for diffing: (kind, phase, normalized message prefix) —
|
||||
* lowercase, whitespace-collapsed, digit runs collapsed (durations, counts,
|
||||
* and tempdir suffixes would otherwise make the same friction land in
|
||||
* "unique to each" across runs), first 80 chars, with redaction applied
|
||||
* FIRST (the caller redacts via redactEntry). Kind IS identity: a delight
|
||||
* and a friction with the same text are different findings, and a
|
||||
* delight→friction flip must surface, never compare equal. Severity is
|
||||
* deliberately EXCLUDED from identity — it is the compared attribute, as a
|
||||
* PER-SEVERITY MULTISET (two errors + one nit vs one error + two nits is a
|
||||
* reported difference even though the severity sets and totals match).
|
||||
*/
|
||||
const IDENTITY_PREFIX_CHARS = 80;
|
||||
|
||||
interface DiffIdentityRecord {
|
||||
kind: string;
|
||||
phase: string;
|
||||
/** Redacted message of the first occurrence (display sample). */
|
||||
message: string;
|
||||
count: number;
|
||||
/** Unique severities, sorted (display); counts live in severity_counts. */
|
||||
severities: string[];
|
||||
severity_counts: Record<string, number>;
|
||||
}
|
||||
|
||||
interface DiffChangedRecord {
|
||||
kind: string;
|
||||
phase: string;
|
||||
message: string;
|
||||
base_count: number;
|
||||
compare_count: number;
|
||||
base_severities: string[];
|
||||
compare_severities: string[];
|
||||
base_severity_counts: Record<string, number>;
|
||||
compare_severity_counts: Record<string, number>;
|
||||
count_changed: boolean;
|
||||
severity_changed: boolean;
|
||||
}
|
||||
|
||||
interface DiffRunBanner {
|
||||
run_id: string;
|
||||
agent?: string;
|
||||
scenario?: string;
|
||||
gbrain_version?: string;
|
||||
interrupted: boolean;
|
||||
/** Malformed JSONL lines skipped by the reader (surfaced, never hidden). */
|
||||
malformed: number;
|
||||
}
|
||||
|
||||
export interface FrictionDiffResult {
|
||||
base: string;
|
||||
compare: string;
|
||||
banner: { base: DiffRunBanner; compare: DiffRunBanner; warnings: string[] };
|
||||
unique_to_base: DiffIdentityRecord[];
|
||||
unique_to_compare: DiffIdentityRecord[];
|
||||
changed: DiffChangedRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a run spec: an exact run-id wins; otherwise treat the spec as an
|
||||
* agent name and pick the LATEST run (listRuns is mtime-sorted, newest first)
|
||||
* whose entries carry that agent — the run-start marker stamps agent on clean
|
||||
* runs, and any agent-stamped entry also counts. Returns undefined if nothing
|
||||
* matches.
|
||||
*/
|
||||
export function resolveRunSpec(spec: string): string | undefined {
|
||||
try {
|
||||
if (existsSync(frictionFile(spec))) return spec;
|
||||
} catch { /* spec has characters a run-id can't; fall through to agent-name resolution */ }
|
||||
for (const run of listRuns()) {
|
||||
try {
|
||||
const { entries } = readFriction(run.runId);
|
||||
if (entries.some(e => e.agent === spec)) return run.runId;
|
||||
} catch { /* unreadable file; skip */ }
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeForIdentity(message: string): string {
|
||||
return message.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ').trim().slice(0, IDENTITY_PREFIX_CHARS);
|
||||
}
|
||||
|
||||
interface IdentityAccum { kind: string; phase: string; message: string; count: number; severities: Map<string, number> }
|
||||
|
||||
/** Sorted-key plain object from a severity count map (deterministic JSON). */
|
||||
function severityCounts(m: Map<string, number>): Record<string, number> {
|
||||
return Object.fromEntries([...m.entries()].sort(([a], [b]) => a.localeCompare(b)));
|
||||
}
|
||||
|
||||
/** Diff operates ONLY on kind friction|delight; markers/interruptions feed the banner. */
|
||||
function collectIdentities(entries: FrictionEntry[]): Map<string, IdentityAccum> {
|
||||
const map = new Map<string, IdentityAccum>();
|
||||
for (const raw of entries) {
|
||||
// Entries from older writers may omit kind; they are friction by contract.
|
||||
const kind = raw.kind ?? 'friction';
|
||||
if (kind !== 'friction' && kind !== 'delight') continue;
|
||||
const e = redactEntry(raw);
|
||||
const key = `${kind}\u0000${e.phase}\u0000${normalizeForIdentity(e.message)}`;
|
||||
let acc = map.get(key);
|
||||
if (!acc) {
|
||||
acc = { kind, phase: e.phase, message: e.message, count: 0, severities: new Map() };
|
||||
map.set(key, acc);
|
||||
}
|
||||
acc.count++;
|
||||
if (e.severity) acc.severities.set(e.severity, (acc.severities.get(e.severity) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function toIdentityRecord(acc: IdentityAccum): DiffIdentityRecord {
|
||||
return {
|
||||
kind: acc.kind,
|
||||
phase: acc.phase,
|
||||
message: acc.message,
|
||||
count: acc.count,
|
||||
severities: [...acc.severities.keys()].sort(),
|
||||
severity_counts: severityCounts(acc.severities),
|
||||
};
|
||||
}
|
||||
|
||||
function bannerFor(runId: string, read: ReadResult): DiffRunBanner {
|
||||
const start = read.entries.find(e => e.kind === 'phase-marker' && e.marker === 'start');
|
||||
const agent = start?.agent ?? read.entries.find(e => e.agent)?.agent;
|
||||
return {
|
||||
run_id: runId,
|
||||
agent,
|
||||
scenario: start?.scenario,
|
||||
gbrain_version: start?.gbrain_version ?? read.entries[0]?.gbrain_version,
|
||||
interrupted: read.entries.some(e => e.kind === 'interrupted'),
|
||||
malformed: read.malformed,
|
||||
};
|
||||
}
|
||||
|
||||
/** Compute the diff between two resolved run-ids. Throws on read errors. */
|
||||
export function computeFrictionDiff(baseRunId: string, compareRunId: string): FrictionDiffResult {
|
||||
const baseRead = readFriction(baseRunId);
|
||||
const compareRead = readFriction(compareRunId);
|
||||
const baseBanner = bannerFor(baseRunId, baseRead);
|
||||
const compareBanner = bannerFor(compareRunId, compareRead);
|
||||
|
||||
const warnings: string[] = [];
|
||||
if ((baseBanner.scenario ?? '') !== (compareBanner.scenario ?? '')) {
|
||||
warnings.push(`scenario differs: ${baseRunId} ran ${baseBanner.scenario ?? '(unknown)'}, ${compareRunId} ran ${compareBanner.scenario ?? '(unknown)'} — entries may not be comparable`);
|
||||
}
|
||||
if ((baseBanner.gbrain_version ?? '') !== (compareBanner.gbrain_version ?? '')) {
|
||||
warnings.push(`gbrain version differs: ${baseRunId} ran ${baseBanner.gbrain_version ?? '(unknown)'}, ${compareRunId} ran ${compareBanner.gbrain_version ?? '(unknown)'}`);
|
||||
}
|
||||
|
||||
const baseIds = collectIdentities(baseRead.entries);
|
||||
const compareIds = collectIdentities(compareRead.entries);
|
||||
const uniqueToBase: DiffIdentityRecord[] = [];
|
||||
const uniqueToCompare: DiffIdentityRecord[] = [];
|
||||
const changed: DiffChangedRecord[] = [];
|
||||
for (const [key, b] of baseIds) {
|
||||
const c = compareIds.get(key);
|
||||
if (!c) { uniqueToBase.push(toIdentityRecord(b)); continue; }
|
||||
const bCounts = severityCounts(b.severities);
|
||||
const cCounts = severityCounts(c.severities);
|
||||
const countChanged = b.count !== c.count;
|
||||
// severity_changed = the per-severity DISTRIBUTION SHAPE changed — a new
|
||||
// severity appeared/disappeared or the mix redistributed (2×error+1×nit →
|
||||
// 1×error+2×nit, which the unique-severity set hides). Uniform scaling
|
||||
// (1×error → 10×error) is purely a count change and count_changed already
|
||||
// reports it. Integer cross-multiplication keeps the proportion test
|
||||
// exact.
|
||||
const bTotal = Object.values(bCounts).reduce((a, n) => a + n, 0);
|
||||
const cTotal = Object.values(cCounts).reduce((a, n) => a + n, 0);
|
||||
let severityChanged = (bTotal === 0) !== (cTotal === 0);
|
||||
if (!severityChanged) {
|
||||
for (const s of new Set([...Object.keys(bCounts), ...Object.keys(cCounts)])) {
|
||||
if ((bCounts[s] ?? 0) * cTotal !== (cCounts[s] ?? 0) * bTotal) { severityChanged = true; break; }
|
||||
}
|
||||
}
|
||||
if (countChanged || severityChanged) {
|
||||
changed.push({
|
||||
kind: b.kind, phase: b.phase, message: b.message,
|
||||
base_count: b.count, compare_count: c.count,
|
||||
base_severities: Object.keys(bCounts), compare_severities: Object.keys(cCounts),
|
||||
base_severity_counts: bCounts, compare_severity_counts: cCounts,
|
||||
count_changed: countChanged, severity_changed: severityChanged,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const [key, c] of compareIds) {
|
||||
if (!baseIds.has(key)) uniqueToCompare.push(toIdentityRecord(c));
|
||||
}
|
||||
const byPhaseThenMessage = (a: { phase: string; message: string }, b: { phase: string; message: string }) =>
|
||||
a.phase.localeCompare(b.phase) || a.message.localeCompare(b.message);
|
||||
uniqueToBase.sort(byPhaseThenMessage);
|
||||
uniqueToCompare.sort(byPhaseThenMessage);
|
||||
changed.sort(byPhaseThenMessage);
|
||||
|
||||
return {
|
||||
base: baseRunId,
|
||||
compare: compareRunId,
|
||||
banner: { base: baseBanner, compare: compareBanner, warnings },
|
||||
unique_to_base: uniqueToBase,
|
||||
unique_to_compare: uniqueToCompare,
|
||||
changed,
|
||||
};
|
||||
}
|
||||
|
||||
function groupByPhase<T extends { phase: string }>(records: T[]): Map<string, T[]> {
|
||||
const map = new Map<string, T[]>();
|
||||
for (const r of records) {
|
||||
if (!map.has(r.phase)) map.set(r.phase, []);
|
||||
map.get(r.phase)!.push(r);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function fmtSeverityCounts(counts: Record<string, number>): string {
|
||||
const parts = Object.entries(counts).map(([s, n]) => (n > 1 ? `${s}×${n}` : s));
|
||||
return parts.length > 0 ? parts.join('/') : '(none)';
|
||||
}
|
||||
|
||||
/** Render the diff as markdown. Sections are labeled by run-id — the diff is
|
||||
* an instrument, not a judge; it never attributes blame to either side. */
|
||||
function renderDiff(diff: FrictionDiffResult): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Friction diff — base \`${diff.base}\` vs compare \`${diff.compare}\``);
|
||||
lines.push('');
|
||||
for (const side of [diff.banner.base, diff.banner.compare]) {
|
||||
const bits = [
|
||||
`agent=${side.agent ?? '(unknown)'}`,
|
||||
`scenario=${side.scenario ?? '(unknown)'}`,
|
||||
`gbrain=${side.gbrain_version ?? '(unknown)'}`,
|
||||
];
|
||||
if (side.interrupted) bits.push('interrupted');
|
||||
if (side.malformed > 0) bits.push(`${side.malformed} malformed line(s) skipped`);
|
||||
lines.push(`- \`${side.run_id}\`: ${bits.join(' · ')}`);
|
||||
}
|
||||
lines.push('');
|
||||
for (const w of diff.banner.warnings) lines.push(`> ⚠ WARN: ${w}`);
|
||||
if (diff.banner.warnings.length > 0) lines.push('');
|
||||
|
||||
if (diff.unique_to_base.length === 0 && diff.unique_to_compare.length === 0 && diff.changed.length === 0) {
|
||||
lines.push('No differences.');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const renderIdentitySection = (title: string, records: DiffIdentityRecord[]) => {
|
||||
lines.push(`## ${title} (${records.length})`);
|
||||
lines.push('');
|
||||
if (records.length === 0) { lines.push('(none)'); lines.push(''); return; }
|
||||
for (const [phase, rs] of groupByPhase(records)) {
|
||||
lines.push(`### \`${phase}\``);
|
||||
lines.push('');
|
||||
for (const r of rs) {
|
||||
const sev = r.severities.length > 0 ? `[${r.severities.join('/')}] ` : '';
|
||||
const kind = r.kind === 'delight' ? '[delight] ' : '';
|
||||
const count = r.count > 1 ? ` ×${r.count}` : '';
|
||||
lines.push(`- ${kind}${sev}${r.message}${count}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
};
|
||||
renderIdentitySection(`Unique to \`${diff.compare}\``, diff.unique_to_compare);
|
||||
renderIdentitySection(`Unique to \`${diff.base}\``, diff.unique_to_base);
|
||||
|
||||
lines.push(`## Shared but changed (${diff.changed.length})`);
|
||||
lines.push('');
|
||||
if (diff.changed.length === 0) { lines.push('(none)'); lines.push(''); }
|
||||
for (const [phase, rs] of groupByPhase(diff.changed)) {
|
||||
lines.push(`### \`${phase}\``);
|
||||
lines.push('');
|
||||
for (const r of rs) {
|
||||
const deltas: string[] = [];
|
||||
if (r.severity_changed) deltas.push(`severity ${fmtSeverityCounts(r.base_severity_counts)} → ${fmtSeverityCounts(r.compare_severity_counts)}`);
|
||||
if (r.count_changed) deltas.push(`count ${r.base_count} → ${r.compare_count}`);
|
||||
const kind = r.kind === 'delight' ? '[delight] ' : '';
|
||||
lines.push(`- ${kind}${r.message} — ${deltas.join('; ')}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function cmdDiff(args: string[]): number {
|
||||
const flags = parseFlags(args);
|
||||
const baseSpec = flags.string('--base');
|
||||
const compareSpec = flags.string('--compare');
|
||||
if (!baseSpec || !compareSpec) {
|
||||
console.error('usage: gbrain friction diff --base <run-or-agent> --compare <run-or-agent> [--json]');
|
||||
return 2;
|
||||
}
|
||||
const baseRun = resolveRunSpec(baseSpec);
|
||||
const compareRun = resolveRunSpec(compareSpec);
|
||||
if (!baseRun || !compareRun) {
|
||||
const unresolved = [!baseRun ? baseSpec : undefined, !compareRun ? compareSpec : undefined]
|
||||
.filter((s): s is string => s !== undefined)
|
||||
.map(s => JSON.stringify(s)).join(', ');
|
||||
const runs = listRuns();
|
||||
const available = runs.length > 0 ? runs.map(r => ` ${r.runId}`).join('\n') : ' (none)';
|
||||
console.error(`friction diff failed: ${unresolved} matched no run-id or agent\navailable runs under ${frictionDir()}:\n${available}`);
|
||||
return 1;
|
||||
}
|
||||
try {
|
||||
const diff = computeFrictionDiff(baseRun, compareRun);
|
||||
process.stdout.write((flags.bool('--json') ? JSON.stringify(diff, null, 2) : renderDiff(diff)) + '\n');
|
||||
return 0;
|
||||
} catch (e) {
|
||||
console.error(`friction diff failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -174,12 +489,16 @@ Subcommands:
|
||||
render Render a run's entries as markdown (default) or JSON
|
||||
list List recent runs with friction/delight counts
|
||||
summary Two-column summary of friction + delight for a run
|
||||
diff Compare two runs: unique-to-each + shared-but-changed entries
|
||||
|
||||
Examples:
|
||||
gbrain friction log --severity confused --phase install --message "init didn't say which engine"
|
||||
gbrain friction render --run-id claw-test-20260428-... --transcripts
|
||||
gbrain friction list --json
|
||||
gbrain friction summary
|
||||
gbrain friction diff --base openclaw --compare hermes --json
|
||||
|
||||
Run-id resolution: --run-id > $GBRAIN_FRICTION_RUN_ID > 'standalone'.`);
|
||||
Run-id resolution: --run-id > $GBRAIN_FRICTION_RUN_ID > 'standalone'.
|
||||
Diff run resolution: an exact run-id wins; otherwise the value is treated as
|
||||
an agent name and resolves to that agent's latest run.`);
|
||||
}
|
||||
|
||||
+12
-3
@@ -38,7 +38,7 @@ import {
|
||||
type SearchMode,
|
||||
type ModeBundle,
|
||||
} from '../core/search/mode.ts';
|
||||
import { readSearchStats } from '../core/search/telemetry.ts';
|
||||
import { readSearchStats, telemetryCoverage, TELEMETRY_COVERAGE_CAVEAT } from '../core/search/telemetry.ts';
|
||||
|
||||
const KNOB_DESCRIPTIONS: Record<keyof ModeBundle, string> = {
|
||||
cache_enabled: 'Semantic query cache on/off',
|
||||
@@ -225,6 +225,7 @@ async function runStatsSubcommand(engine: BrainEngine, args: string[]): Promise<
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 2,
|
||||
...stats,
|
||||
coverage: telemetryCoverage(),
|
||||
graph_signals: gsSection,
|
||||
_meta: {
|
||||
metric_glossary: {
|
||||
@@ -241,11 +242,15 @@ async function runStatsSubcommand(engine: BrainEngine, args: string[]): Promise<
|
||||
}
|
||||
|
||||
console.log(`Search stats over the last ${stats.window_days} days:`);
|
||||
console.log(` Coverage note: ${TELEMETRY_COVERAGE_CAVEAT}`);
|
||||
console.log('');
|
||||
console.log(` Total searches: ${stats.total_calls}`);
|
||||
if (stats.total_calls === 0) {
|
||||
console.log('');
|
||||
console.log('No telemetry recorded yet. Run a few `gbrain query` calls and re-check.');
|
||||
console.log('No telemetry recorded in this window. This can mean no search activity, or');
|
||||
console.log('it can reflect the coverage gap above — a lone short-lived CLI call is often');
|
||||
console.log('not enough to trigger a flush. `gbrain serve` / an MCP session is more likely');
|
||||
console.log('to record counts over time (telemetry stays best-effort either way).');
|
||||
// Still print the graph-signals section since failures are tracked
|
||||
// independently of the search_telemetry table.
|
||||
if (gsSection.enabled || gsSection.failures_count > 0) {
|
||||
@@ -382,6 +387,7 @@ async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<v
|
||||
schema_version: 2,
|
||||
status: 'insufficient_data',
|
||||
total_calls: stats.total_calls,
|
||||
coverage: telemetryCoverage(),
|
||||
recommendations: [],
|
||||
message: 'Not enough search activity in the last 7 days to tune. Run `gbrain search stats` after some real usage.',
|
||||
}, null, 2));
|
||||
@@ -389,7 +395,8 @@ async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<v
|
||||
}
|
||||
console.log('Not enough search activity in the last 7 days to tune.');
|
||||
console.log(`Total searches: ${stats.total_calls} (need >= 20 for confident recommendations).`);
|
||||
console.log('Run a few `gbrain query` calls, then re-run `gbrain search tune`.');
|
||||
console.log(`(${TELEMETRY_COVERAGE_CAVEAT} Low counts can reflect this gap, not just low usage.)`);
|
||||
console.log('Use `gbrain serve` or an MCP session for a while, then re-run `gbrain search tune`.');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -448,6 +455,7 @@ async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<v
|
||||
total_calls: stats.total_calls,
|
||||
cache_hit_rate: stats.cache_hit_rate,
|
||||
active_mode: resolved.resolved_mode,
|
||||
coverage: telemetryCoverage(),
|
||||
recommendations: recs,
|
||||
applied: apply ? recs.map(r => r.apply_command) : [],
|
||||
_meta: {
|
||||
@@ -466,6 +474,7 @@ async function runTuneSubcommand(engine: BrainEngine, args: string[]): Promise<v
|
||||
}
|
||||
|
||||
console.log(`Search tune (last 7 days, active mode: ${resolved.resolved_mode}):`);
|
||||
console.log(`(${TELEMETRY_COVERAGE_CAVEAT})`);
|
||||
console.log('');
|
||||
|
||||
if (recs.length === 0) {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/**
|
||||
* AgentRunner — pluggable contract for invoking external agents (openclaw,
|
||||
* hermes, codex, …) inside the claw-test harness. v1 ships a single
|
||||
* implementation (openclaw); the interface stays narrow and concrete so
|
||||
* adding a second runner in v1.1 is a ~50-line file.
|
||||
* hermes, codex, …) inside the claw-test harness. Two implementations ship
|
||||
* (openclaw, hermes); the interface stays narrow and concrete so adding
|
||||
* another runner is a ~100-line file.
|
||||
*
|
||||
* The harness wraps spawn/timeout/transcript-capture; runners only have to
|
||||
* answer "where's your binary?" and "how do I invoke it with this prompt?".
|
||||
@@ -82,6 +82,50 @@ export interface TranscriptEvent {
|
||||
bytes: Buffer;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared runner helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Env keys every runner forwards to its agent subprocess. Runners compose
|
||||
* `[...BASE_ENV_ALLOWLIST, ...delta]` instead of duplicating the list — the
|
||||
* allowlist (not a denylist) is the leak barrier: anything not named here
|
||||
* never reaches the agent.
|
||||
*
|
||||
* GBRAIN_DATABASE_URL is deliberately ABSENT (removed in the hermes-harness
|
||||
* wave's adversarial review): live mode's staging + success oracle operate on
|
||||
* the hermetic PGLite under GBRAIN_HOME=tempdir, and an inherited
|
||||
* GBRAIN_DATABASE_URL would flip only the AGENT's gbrain children to the
|
||||
* operator's real Postgres — polluting the real brain while the oracle probes
|
||||
* the untouched PGLite and fails with a misleading verdict.
|
||||
*/
|
||||
export const BASE_ENV_ALLOWLIST = [
|
||||
'PATH', 'HOME', 'USER', 'LANG', 'TZ', 'NODE_ENV',
|
||||
'ANTHROPIC_API_KEY', 'OPENAI_API_KEY',
|
||||
'GBRAIN_HOME', 'GBRAIN_FRICTION_RUN_ID',
|
||||
// Proxy plumbing (both spellings — Node reads upper, Python/curl read
|
||||
// lower): an operator behind a corporate proxy runs their agent through
|
||||
// these, and dropping them turns live mode into a misleading network
|
||||
// failure blamed on the agent.
|
||||
'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY',
|
||||
'http_proxy', 'https_proxy', 'no_proxy',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Validate a *_BIN env override: must be absolute, free of `..` segments, and
|
||||
* free of shell-active characters. The value is interpolated into generated
|
||||
* sh shim scripts (single-quoted), so quotes/backslashes/dollar/backtick or a
|
||||
* newline would break out of the quoting and become code — reject them
|
||||
* outright rather than trying to escape. Spaces are fine (quoted).
|
||||
* Returns an error string (naming the env var) or null when valid.
|
||||
*/
|
||||
export function validateBinPathEnv(envName: string, p: string): string | null {
|
||||
if (!p.startsWith('/')) return `${envName} must be absolute; got ${p}`;
|
||||
if (p.split('/').includes('..')) return `${envName} must not contain '..' segments; got ${p}`;
|
||||
if (/['"`$\\\n\r]/.test(p)) return `${envName} must not contain quotes, backslashes, dollar signs, backticks, or newlines; got ${p}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Registry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Hermes runner — invokes the real `hermes` binary (NousResearch
|
||||
* hermes-agent, the public platform) in a tempdir with a BRIEF.md prompt.
|
||||
* Live mode only.
|
||||
*
|
||||
* Invocation pattern (verified against a pinned local install, v0.20.0 —
|
||||
* see docs/mcp/HERMES-CLI-PIN.md):
|
||||
* hermes -z "<brief>"
|
||||
*
|
||||
* The z flag is Hermes's headless one-shot: single prompt in, final response
|
||||
* text on stdout, nothing else. We deliberately do NOT pass Hermes's
|
||||
* working-directory flag (spelled "in") — `spawnWithCapture` already sets
|
||||
* `cwd`. `opts.agentName` is unused: the one-shot mode has no sub-agent
|
||||
* selector.
|
||||
*
|
||||
* Hermeticity posture (deliberate): live mode runs the OPERATOR's configured
|
||||
* Hermes — the real ~/.hermes (model settings, skills, sessions) is inherited
|
||||
* unless HERMES_HOME points elsewhere — against a hermetic BRAIN. The fully
|
||||
* hermetic lane is the door e2e (install-real-hermes.serial.test.ts).
|
||||
*
|
||||
* Binary resolution: $HERMES_BIN > `which hermes` > unavailable.
|
||||
* Path validation: must be absolute, must be executable, no '..' segments.
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { statSync } from 'fs';
|
||||
import {
|
||||
BASE_ENV_ALLOWLIST,
|
||||
validateBinPathEnv,
|
||||
type AgentRunner,
|
||||
type DetectResult,
|
||||
type InvokeOpts,
|
||||
type InvokeResult,
|
||||
} from '../agent-runner.ts';
|
||||
import { spawnWithCapture } from '../transcript-capture.ts';
|
||||
|
||||
/**
|
||||
* Allow-list for env propagation when spawning hermes. Delta from the shared
|
||||
* base: HERMES_HOME, so callers (door e2e, CI) can point Hermes at an
|
||||
* isolated home instead of the operator's real ~/.hermes; and
|
||||
* OPENROUTER_API_KEY, because OpenRouter is a Hermes-documented auth path
|
||||
* (docs/mcp/HERMES.md) — an operator whose hermes auths only via that env var
|
||||
* would otherwise see "no inference provider" and the harness would blame the
|
||||
* agent. Caveat (observed, docs/mcp/HERMES-CLI-PIN.md): live mode forwards
|
||||
* whatever provider keys the operator's shell exports, mirroring a direct
|
||||
* hermes run — with MULTIPLE keys visible and no model pinned in config,
|
||||
* Hermes's provider auto-routing can mis-route and fail with an HTTP 401 in
|
||||
* the final text. The operator's own config.yaml model pin is what prevents
|
||||
* that, same as it does outside the harness.
|
||||
*/
|
||||
const ENV_ALLOWLIST = [...BASE_ENV_ALLOWLIST, 'HERMES_HOME', 'OPENROUTER_API_KEY'];
|
||||
|
||||
export class HermesRunner implements AgentRunner {
|
||||
readonly name = 'hermes';
|
||||
|
||||
async detect(): Promise<DetectResult> {
|
||||
const fromEnv = process.env.HERMES_BIN?.trim();
|
||||
let binPath: string | undefined;
|
||||
|
||||
if (fromEnv) {
|
||||
const validation = validateBinPathEnv('HERMES_BIN', fromEnv);
|
||||
if (validation) return { available: false, reason: validation };
|
||||
binPath = fromEnv;
|
||||
} else {
|
||||
try {
|
||||
const out = execSync('which hermes', { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
const found = out.trim();
|
||||
if (!found || !found.startsWith('/')) {
|
||||
return { available: false, reason: 'hermes not on PATH' };
|
||||
}
|
||||
binPath = found;
|
||||
} catch {
|
||||
return { available: false, reason: 'hermes not on PATH' };
|
||||
}
|
||||
}
|
||||
|
||||
if (!binPath) return { available: false, reason: 'no binary resolved' };
|
||||
|
||||
try {
|
||||
const s = statSync(binPath);
|
||||
if (!s.isFile()) return { available: false, reason: `not a regular file: ${binPath}` };
|
||||
// eslint-disable-next-line no-bitwise
|
||||
if (!(s.mode & 0o111)) return { available: false, reason: `not executable: ${binPath}` };
|
||||
} catch (e) {
|
||||
return { available: false, reason: `stat failed: ${e instanceof Error ? e.message : String(e)}` };
|
||||
}
|
||||
|
||||
return { available: true, binPath };
|
||||
}
|
||||
|
||||
async invoke(opts: InvokeOpts): Promise<InvokeResult> {
|
||||
const detected = await this.detect();
|
||||
if (!detected.available || !detected.binPath) {
|
||||
throw new Error(`hermes runner unavailable: ${detected.reason ?? 'unknown'}`);
|
||||
}
|
||||
const args = ['-z', opts.brief];
|
||||
|
||||
// Filter env to allow-list, then merge caller overrides.
|
||||
const baseEnv: Record<string, string> = {};
|
||||
for (const key of ENV_ALLOWLIST) {
|
||||
const v = process.env[key];
|
||||
if (typeof v === 'string') baseEnv[key] = v;
|
||||
}
|
||||
const env: Record<string, string> = { ...baseEnv, ...opts.env };
|
||||
|
||||
const result = await spawnWithCapture(detected.binPath, args, {
|
||||
cwd: opts.cwd,
|
||||
env,
|
||||
timeoutMs: opts.timeoutMs,
|
||||
transcriptSink: opts.transcriptSink,
|
||||
});
|
||||
|
||||
return { exitCode: result.exitCode, durationMs: result.durationMs };
|
||||
}
|
||||
}
|
||||
@@ -15,16 +15,19 @@
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { statSync } from 'fs';
|
||||
import type { AgentRunner, DetectResult, InvokeOpts, InvokeResult } from '../agent-runner.ts';
|
||||
import {
|
||||
BASE_ENV_ALLOWLIST,
|
||||
validateBinPathEnv,
|
||||
type AgentRunner,
|
||||
type DetectResult,
|
||||
type InvokeOpts,
|
||||
type InvokeResult,
|
||||
} from '../agent-runner.ts';
|
||||
import { spawnWithCapture } from '../transcript-capture.ts';
|
||||
|
||||
const DEFAULT_AGENT_NAME = 'default';
|
||||
/** Allow-list for env propagation when spawning openclaw. */
|
||||
const ENV_ALLOWLIST = [
|
||||
'PATH', 'HOME', 'USER', 'LANG', 'TZ', 'NODE_ENV',
|
||||
'ANTHROPIC_API_KEY', 'OPENAI_API_KEY',
|
||||
'GBRAIN_HOME', 'GBRAIN_FRICTION_RUN_ID', 'GBRAIN_DATABASE_URL',
|
||||
];
|
||||
/** Allow-list for env propagation when spawning openclaw (no delta from base). */
|
||||
const ENV_ALLOWLIST = [...BASE_ENV_ALLOWLIST];
|
||||
|
||||
export class OpenClawRunner implements AgentRunner {
|
||||
readonly name = 'openclaw';
|
||||
@@ -34,7 +37,7 @@ export class OpenClawRunner implements AgentRunner {
|
||||
let binPath: string | undefined;
|
||||
|
||||
if (fromEnv) {
|
||||
const validation = validateAbsolutePath(fromEnv);
|
||||
const validation = validateBinPathEnv('OPENCLAW_BIN', fromEnv);
|
||||
if (validation) return { available: false, reason: validation };
|
||||
binPath = fromEnv;
|
||||
} else {
|
||||
@@ -91,8 +94,3 @@ export class OpenClawRunner implements AgentRunner {
|
||||
}
|
||||
}
|
||||
|
||||
function validateAbsolutePath(p: string): string | null {
|
||||
if (!p.startsWith('/')) return `OPENCLAW_BIN must be absolute; got ${p}`;
|
||||
if (p.split('/').includes('..')) return `OPENCLAW_BIN must not contain '..' segments; got ${p}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -14,6 +14,22 @@ import { fileURLToPath } from 'url';
|
||||
|
||||
export type ScenarioKind = 'fresh-install' | 'upgrade';
|
||||
|
||||
/**
|
||||
* Live-mode success oracle, declared per scenario. The harness verifies these
|
||||
* AFTER the agent exits 0 — exit code alone would pass an agent that did
|
||||
* nothing. All fields optional; a scenario with no oracle gets the kind's
|
||||
* default verification (fresh-install: doctor only; upgrade: schema-version
|
||||
* probe must ADVANCE during the agent turn).
|
||||
*/
|
||||
export interface ScenarioOracle {
|
||||
/** Query the staged brain must answer post-run (live mode). */
|
||||
query?: string;
|
||||
/** Minimum result count for `query` (default 1 when query is set). */
|
||||
minResults?: number;
|
||||
/** Workspace-relative paths that must exist post-run (proves brief steps ran). */
|
||||
filesExist?: string[];
|
||||
}
|
||||
|
||||
export interface ScenarioConfig {
|
||||
/** Directory the scenario was loaded from. Always absolute. */
|
||||
dir: string;
|
||||
@@ -33,6 +49,8 @@ export interface ScenarioConfig {
|
||||
brainRelative?: string;
|
||||
/** Path to seed dir for upgrade scenarios. */
|
||||
seedRelative?: string;
|
||||
/** Live-mode success oracle (see ScenarioOracle). */
|
||||
oracle?: ScenarioOracle;
|
||||
}
|
||||
|
||||
/** Default fixtures root, override via $GBRAIN_CLAW_SCENARIOS_DIR for tests. */
|
||||
@@ -61,10 +79,30 @@ export function listScenarios(root?: string): string[] {
|
||||
.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Confine a scenario-declared relative path inside the scenario dir. The
|
||||
* scenario pack is a trust boundary (loadable from arbitrary dirs via the
|
||||
* scenarios-dir env override, and its content flows to an external agent):
|
||||
* a traversal-shaped brief/brain/seed would let a pack read — and, via the
|
||||
* BRIEF, exfiltrate to the agent — arbitrary operator files.
|
||||
*/
|
||||
function confineToScenarioDir(name: string, dir: string, rel: string, field: string): void {
|
||||
const abs = resolve(dir, rel);
|
||||
if (abs !== dir && !abs.startsWith(dir + '/')) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: ${field} must stay inside the scenario dir; got ${JSON.stringify(rel)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Load and validate one scenario by name. */
|
||||
export function loadScenario(name: string, root?: string): ScenarioConfig {
|
||||
// The name is a directory segment — never a path. A traversal-shaped name
|
||||
// would resolve scenario.json (and everything the scenario references)
|
||||
// outside the fixtures root.
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) || name.includes('..')) {
|
||||
throw new Error(`scenario name ${JSON.stringify(name)} not found (names are letters, digits, dot, dash, underscore)`);
|
||||
}
|
||||
const r = root ?? defaultFixturesRoot();
|
||||
const dir = join(r, name);
|
||||
const dir = resolve(join(r, name));
|
||||
const cfgPath = join(dir, 'scenario.json');
|
||||
if (!existsSync(cfgPath)) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)} not found at ${cfgPath}`);
|
||||
@@ -86,6 +124,7 @@ export function loadScenario(name: string, root?: string): ScenarioConfig {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: expected_phases must be a string[]`);
|
||||
}
|
||||
const briefRel = typeof cfg.brief === 'string' ? cfg.brief : 'BRIEF.md';
|
||||
confineToScenarioDir(name, dir, briefRel, 'brief');
|
||||
if (!existsSync(join(dir, briefRel))) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: BRIEF.md missing at ${briefRel}`);
|
||||
}
|
||||
@@ -98,8 +137,60 @@ export function loadScenario(name: string, root?: string): ScenarioConfig {
|
||||
};
|
||||
if (typeof cfg.from_version === 'string') out.fromVersion = cfg.from_version;
|
||||
if (typeof cfg.description === 'string') out.description = cfg.description;
|
||||
if (typeof cfg.brain === 'string') out.brainRelative = cfg.brain;
|
||||
if (typeof cfg.seed === 'string') out.seedRelative = cfg.seed;
|
||||
if (typeof cfg.brain === 'string') {
|
||||
confineToScenarioDir(name, dir, cfg.brain, 'brain');
|
||||
out.brainRelative = cfg.brain;
|
||||
}
|
||||
if (typeof cfg.seed === 'string') {
|
||||
confineToScenarioDir(name, dir, cfg.seed, 'seed');
|
||||
out.seedRelative = cfg.seed;
|
||||
}
|
||||
if (cfg.oracle !== undefined) {
|
||||
if (!cfg.oracle || typeof cfg.oracle !== 'object' || Array.isArray(cfg.oracle)) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle must be a JSON object`);
|
||||
}
|
||||
const o = cfg.oracle as Record<string, unknown>;
|
||||
const oracle: ScenarioOracle = {};
|
||||
if (o.query !== undefined) {
|
||||
if (typeof o.query !== 'string' || !o.query.trim()) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.query must be a non-empty string`);
|
||||
}
|
||||
// The query lands in gbrain argv — a leading dash would parse as a CLI
|
||||
// flag instead of a query (scenario packs load from arbitrary dirs via
|
||||
// the scenarios-dir env override, so treat this as a trust boundary).
|
||||
if (o.query.trim().startsWith('-')) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.query must not start with a dash`);
|
||||
}
|
||||
oracle.query = o.query;
|
||||
oracle.minResults = 1;
|
||||
}
|
||||
if (o.min_results !== undefined) {
|
||||
// Rejecting min_results without query keeps the config honest: the
|
||||
// harness only reads minResults when a query is declared, and accepting
|
||||
// config it never enforces would be a silent no-op.
|
||||
if (o.query === undefined) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.min_results requires oracle.query`);
|
||||
}
|
||||
if (typeof o.min_results !== 'number' || !Number.isFinite(o.min_results) || o.min_results < 0) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.min_results must be a number >= 0`);
|
||||
}
|
||||
oracle.minResults = o.min_results;
|
||||
}
|
||||
if (o.files_exist !== undefined) {
|
||||
if (!Array.isArray(o.files_exist) || !o.files_exist.every(x => typeof x === 'string' && x.trim())) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.files_exist must be a string[]`);
|
||||
}
|
||||
// Paths are resolved relative to the run's workspace; confine them so a
|
||||
// scenario pack can't probe arbitrary operator paths.
|
||||
for (const p of o.files_exist as string[]) {
|
||||
if (p.startsWith('/') || p.split('/').includes('..')) {
|
||||
throw new Error(`scenario ${JSON.stringify(name)}: oracle.files_exist entries must be workspace-relative (no absolute paths, no '..'): ${JSON.stringify(p)}`);
|
||||
}
|
||||
}
|
||||
oracle.filesExist = o.files_exist as string[];
|
||||
}
|
||||
out.oracle = oracle;
|
||||
}
|
||||
// Default brain path conventions
|
||||
if (!out.brainRelative && existsSync(join(dir, 'brain'))) out.brainRelative = 'brain';
|
||||
if (!out.seedRelative && out.kind === 'upgrade' && existsSync(join(dir, 'seed'))) {
|
||||
|
||||
@@ -66,6 +66,34 @@ export async function seedPgliteFromFile(opts: { dbPath: string; sqlPath: string
|
||||
return seedPglite({ dbPath: opts.dbPath, sql });
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-mutating schema-version probe for the live-mode upgrade oracle.
|
||||
*
|
||||
* The migration runner records its position via setConfig('version', …), i.e.
|
||||
* `config.key = 'version'`. Reading it through the normal CLI is NOT valid as
|
||||
* an upgrade oracle: every CLI connect routes through connectEngine → initSchema,
|
||||
* which auto-applies pending migrations — the verifier would perform the very
|
||||
* upgrade a do-nothing agent skipped. This probe opens PGLite directly (same
|
||||
* path as the seeder) and reads the row without touching the migration chain.
|
||||
*
|
||||
* Returns the recorded version, or null when the db/table/row doesn't exist.
|
||||
*/
|
||||
export async function readPgliteSchemaVersion(dbPath: string): Promise<number | null> {
|
||||
if (!existsSync(dbPath)) return null;
|
||||
const engine = new PGLiteEngine();
|
||||
try {
|
||||
await engine.connect({ engine: 'pglite', database_path: dbPath });
|
||||
const rows = await (engine as any).db.query(`SELECT value FROM config WHERE key = 'version'`);
|
||||
const value = rows?.rows?.[0]?.value;
|
||||
const n = typeof value === 'string' ? Number.parseInt(value, 10) : typeof value === 'number' ? value : NaN;
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
try { await engine.disconnect(); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a SQL dump into individual statements. Naïve `;` split that respects
|
||||
* single-quoted strings and `--` line comments. Sufficient for canonical
|
||||
|
||||
@@ -105,6 +105,8 @@ export interface SpawnResult {
|
||||
}
|
||||
|
||||
const SIGTERM_GRACE_MS = 5_000;
|
||||
/** After 'exit', how long to wait for 'close' (pipe drain) before resolving anyway. */
|
||||
const STREAM_DRAIN_GRACE_MS = 2_000;
|
||||
|
||||
export async function spawnWithCapture(bin: string, args: string[], opts: SpawnOpts): Promise<SpawnResult> {
|
||||
const start = Date.now();
|
||||
@@ -116,19 +118,33 @@ export async function spawnWithCapture(bin: string, args: string[], opts: SpawnO
|
||||
env: opts.env,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
shell: false,
|
||||
// Own process group so timeout kills reach grandchildren too — agents
|
||||
// spawn MCP servers and gbrain children; signalling only the direct
|
||||
// PID leaves those orphaned holding the stdout/stderr pipes open.
|
||||
detached: true,
|
||||
});
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
return;
|
||||
}
|
||||
|
||||
// Signal the whole process group (negative pid); fall back to the direct
|
||||
// child if the group is already gone or grouping failed.
|
||||
const killTree = (sig: NodeJS.Signals) => {
|
||||
const pid = child.pid;
|
||||
if (pid) {
|
||||
try { process.kill(-pid, sig); return; } catch { /* group gone or not a leader */ }
|
||||
}
|
||||
try { child.kill(sig); } catch { /* already gone */ }
|
||||
};
|
||||
|
||||
let timedOut = false;
|
||||
let killTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const wallClockTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
try { child.kill('SIGTERM'); } catch { /* already gone */ }
|
||||
killTree('SIGTERM');
|
||||
killTimer = setTimeout(() => {
|
||||
try { child.kill('SIGKILL'); } catch { /* already gone */ }
|
||||
killTree('SIGKILL');
|
||||
}, SIGTERM_GRACE_MS);
|
||||
}, opts.timeoutMs);
|
||||
|
||||
@@ -151,15 +167,17 @@ export async function spawnWithCapture(bin: string, args: string[], opts: SpawnO
|
||||
reject(e);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// No payload: close stdin anyway. An agent build that waits for stdin
|
||||
// EOF would otherwise block silently until the wall-clock kill — a paid
|
||||
// live turn burned as a timeout.
|
||||
try { child.stdin?.end(); } catch { /* stream already gone */ }
|
||||
}
|
||||
|
||||
child.on('error', (err) => {
|
||||
clearTimeout(wallClockTimer);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
let settled = false;
|
||||
const settle = (code: number | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(wallClockTimer);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
resolve({
|
||||
@@ -167,6 +185,25 @@ export async function spawnWithCapture(bin: string, args: string[], opts: SpawnO
|
||||
durationMs: Date.now() - start,
|
||||
timedOut,
|
||||
});
|
||||
};
|
||||
|
||||
child.on('error', (err) => {
|
||||
clearTimeout(wallClockTimer);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
// 'close' (all pipes drained) is the clean path. But a grandchild that
|
||||
// survives the group kill can hold the pipes open forever, so 'exit' arms
|
||||
// a short drain grace and then settles regardless — the harness must not
|
||||
// hang AFTER its own timeout already fired.
|
||||
child.on('close', (code) => settle(code));
|
||||
child.on('exit', (code) => {
|
||||
const t = setTimeout(() => settle(code), STREAM_DRAIN_GRACE_MS);
|
||||
t.unref?.();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'check-backlinks': ['--background', '--brain', '--brain-wide-max-cost-usd', '--dir', '--dry-run', '--explain', '--follow', '--help', '--include-frontmatter', '--json', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--source', '--stale', '--timeout', '--type'],
|
||||
'check-resolvable': ['--brain', '--dry-run', '--fix', '--help', '--json', '--skills-dir', '--source', '--strict', '--verbose'],
|
||||
'check-update': ['--all', '--brain', '--check', '--dim', '--ff-only', '--help', '--json', '--markdown', '--migrate-only', '--non-interactive', '--refresh-cache', '--source', '--swap-only', '--to', '--version', '--yes'],
|
||||
'claw-test': ['--agent', '--brain', '--dir', '--help', '--json', '--keep-tempdir', '--list-agents', '--live', '--local', '--message', '--no-embed', '--no-embedding', '--path', '--pglite', '--progress-json', '--prompt-file', '--run-id', '--scenario', '--source', '--transcripts'],
|
||||
'claw-test': ['--ab', '--agent', '--all', '--auto-update', '--brain', '--break-lock', '--build-index', '--by-mention', '--compile', '--days', '--dir', '--exclusive', '--force-retry', '--force-schema', '--from-meetings', '--help', '--history', '--http', '--json', '--keep-tempdir', '--lang', '--list-agents', '--live', '--local', '--locks', '--markdown', '--max-age', '--message', '--multimodal', '--no-embed', '--no-embedding', '--no-extract', '--path', '--pglite', '--phase', '--priority', '--progress-json', '--prompt-file', '--refresh-unqualified', '--remediate', '--rollback', '--run-id', '--scenario', '--skip-verify', '--source', '--stale', '--transcripts', '--undo-wave', '--use-captured-snapshot', '--with-calibration', '--yes'],
|
||||
'code-callees': ['--aliases', '--all', '--all-sources', '--brain', '--chunker-debug', '--clone-dir', '--confirm-destructive', '--federated', '--force', '--help', '--include-null-signature', '--json', '--limit', '--no-extract', '--no-federated', '--no-json', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--stale', '--supersessions', '--thin', '--url', '--url-managed', '--yes'],
|
||||
'code-callers': ['--aliases', '--all', '--all-sources', '--brain', '--chunker-debug', '--clone-dir', '--confirm-destructive', '--federated', '--force', '--help', '--include-null-signature', '--json', '--limit', '--no-extract', '--no-federated', '--no-json', '--path', '--pattern', '--pending', '--repo', '--reset', '--resolve', '--restore-only', '--source', '--stale', '--supersessions', '--thin', '--url', '--url-managed', '--yes'],
|
||||
'code-def': ['--aliases', '--all', '--brain', '--chunker-debug', '--help', '--include-null-signature', '--json', '--lang', '--limit', '--no-extract', '--no-json', '--pattern', '--pending', '--pretty', '--reset', '--resolve', '--source', '--stale', '--supersessions', '--thin', '--yes'],
|
||||
@@ -47,7 +47,7 @@ export const CLI_FLAG_REGISTRY: Record<string, readonly string[]> = {
|
||||
'files': ['--aliases', '--all', '--background', '--brain', '--brain-wide-max-cost-usd', '--dry-run', '--explain', '--fast', '--follow', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--no-embedding', '--no-extract', '--no-pointer', '--page', '--pattern', '--pending', '--progress-interval', '--progress-json', '--quiet', '--remediate', '--reset', '--resolve', '--retry-failed', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--type', '--yes'],
|
||||
'forget': ['--aliases', '--all', '--allow-empty', '--apply', '--as-context', '--asof', '--auto', '--bound-max-concurrent', '--bound-slug-prefixes', '--bound-source', '--bound-tools', '--brain', '--budget-tokens', '--budget-usd-per-day', '--by-mention', '--content', '--date', '--days', '--entities', '--fast', '--federated', '--file', '--follow', '--force', '--from-pages', '--grep', '--help', '--http', '--image', '--include-expired', '--include-null-signature', '--json', '--kind', '--limit', '--max-usd', '--mcp-only', '--mode', '--multimodal', '--no-embedding', '--no-extract', '--no-federated', '--offset', '--path', '--pattern', '--pending', '--progress-interval', '--progress-json', '--query', '--quiet', '--reason', '--repo', '--reset', '--resolve', '--restore-only', '--rollup', '--save', '--session', '--session-id', '--since', '--since-last-run', '--slug', '--slugs', '--source', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--today', '--token-ttl', '--trusted-extraction', '--url', '--watch', '--with-db', '--yes'],
|
||||
'founder': ['--aliases', '--all', '--brain', '--fast', '--force', '--from-pages', '--help', '--http', '--include-null-signature', '--json', '--mcp-only', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--since', '--source', '--stale', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--until'],
|
||||
'friction': ['--agent', '--brain', '--help', '--hint', '--json', '--kind', '--message', '--no-redact', '--phase', '--redact', '--run-id', '--severity', '--source', '--transcript-path', '--transcripts'],
|
||||
'friction': ['--agent', '--base', '--brain', '--compare', '--help', '--hint', '--json', '--kind', '--message', '--no-redact', '--phase', '--redact', '--run-id', '--severity', '--source', '--transcript-path', '--transcripts'],
|
||||
'frontmatter': ['--aliases', '--all', '--allow-catch-all', '--brain', '--cached', '--diff-filter', '--dry-run', '--exclude-standard', '--fast', '--fix', '--force', '--from-pages', '--get', '--help', '--http', '--include-catch-all', '--include-null-signature', '--json', '--name-only', '--name-status', '--no-embedding', '--no-extract', '--no-verify', '--others', '--pattern', '--pending', '--reset', '--resolve', '--source', '--stale', '--strategy', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl', '--uninstall', '--write-back'],
|
||||
'graph-query': ['--aliases', '--all', '--brain', '--depth', '--direction', '--explain', '--fast', '--force', '--from-pages', '--help', '--http', '--include-foreign', '--include-null-signature', '--json', '--lang', '--markdown', '--mcp-only', '--multimodal', '--near-symbol', '--no-embedding', '--no-extract', '--pattern', '--pending', '--reset', '--resolve', '--restore-only', '--source', '--stale', '--supersessions', '--surface', '--symbol-kind', '--thin', '--timeout', '--token-ttl', '--type'],
|
||||
'hook': ['--aliases', '--all', '--allow-unverified-remote', '--batch-limit', '--brain', '--budget-ms', '--cached', '--count', '--delete-brain', '--detach', '--diff-filter', '--end-of-options', '--env', '--exclude-standard', '--fast', '--force', '--from-pages', '--get', '--harness', '--help', '--http', '--include-null-signature', '--jq', '--json', '--name-only', '--no-embedding', '--no-extract', '--once', '--others', '--path', '--pattern', '--pending', '--porcelain', '--quiet', '--reset', '--resolve', '--show-current', '--show-toplevel', '--source', '--stale', '--stats', '--supersessions', '--surface', '--thin', '--timeout', '--token-ttl'],
|
||||
|
||||
@@ -25,7 +25,7 @@ import type { BrainEngine } from '../engine.ts';
|
||||
import type { PhaseResult, PhaseError } from '../cycle.ts';
|
||||
import { MinionQueue } from '../minions/queue.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, SubagentHandlerData } from '../minions/types.ts';
|
||||
import type { MinionJobInput, MinionJobStatus, SubagentHandlerData } from '../minions/types.ts';
|
||||
import { serializeMarkdown } from '../markdown.ts';
|
||||
import type { Page, PageType } from '../types.ts';
|
||||
// #2415: allow-list + output-root resolution shared with the synthesize
|
||||
@@ -221,7 +221,7 @@ export async function runPhasePatterns(
|
||||
// parent job otherwise deadlocks a fully-occupied worker (#2050).
|
||||
await runSubagentsInline(engine, queue, childQueueName, opts.yieldDuringPhase);
|
||||
|
||||
let outcome: string;
|
||||
let outcome: MinionJobStatus | 'timeout';
|
||||
try {
|
||||
const final = await waitForCompletion(queue, job.id, {
|
||||
timeoutMs: budgets.waitTimeoutMs,
|
||||
@@ -271,7 +271,7 @@ export async function runPhasePatterns(
|
||||
// returned status:ok even when the subagent timed out (e.g. no
|
||||
// subagent-capable worker slot free for the whole wait window) and zero
|
||||
// pattern pages were written — a silent no-op for days.
|
||||
if (outcome !== 'complete') {
|
||||
if (outcome !== 'completed') {
|
||||
if (writtenRefs.length === 0) {
|
||||
return {
|
||||
phase: 'patterns',
|
||||
|
||||
+14
-4
@@ -1,11 +1,13 @@
|
||||
/**
|
||||
* Friction reporter — JSONL-backed signal capture for the claw-test feedback loop.
|
||||
*
|
||||
* The friction CLI (`gbrain friction log/render/list/summary`) writes here.
|
||||
* The claw-test harness reads here. The agent calls `gbrain friction log`
|
||||
* directly when it hits something confusing, missing, or wrong.
|
||||
* The friction CLI (`gbrain friction log/render/list/summary/diff`) writes
|
||||
* and reads here. The claw-test harness reads here. The agent calls
|
||||
* `gbrain friction log` directly when it hits something confusing, missing,
|
||||
* or wrong; `friction diff` compares two runs cross-agent.
|
||||
*
|
||||
* Storage shape: append-only JSONL files under `$GBRAIN_HOME/friction/`.
|
||||
* Storage shape: append-only JSONL files under `$GBRAIN_HOME/.gbrain/friction/`
|
||||
* (configDir() appends the '.gbrain' segment).
|
||||
* - `<run-id>.jsonl` for each harness run (run-id from $GBRAIN_FRICTION_RUN_ID)
|
||||
* - `standalone.jsonl` for entries logged outside a harness run
|
||||
*
|
||||
@@ -61,6 +63,10 @@ export interface FrictionEntry {
|
||||
transcript_offset?: number;
|
||||
/** For phase-marker entries only. */
|
||||
marker?: PhaseMarker;
|
||||
/** Scenario name the run executed (stamped on the run-start marker). */
|
||||
scenario?: string;
|
||||
/** Harness meta-record schema version (stamped on the run-start marker). */
|
||||
harness_schema?: number;
|
||||
}
|
||||
|
||||
export interface FrictionLogInput {
|
||||
@@ -74,6 +80,8 @@ export interface FrictionLogInput {
|
||||
agent?: string;
|
||||
transcriptOffset?: number;
|
||||
marker?: PhaseMarker;
|
||||
scenario?: string;
|
||||
harnessSchema?: number;
|
||||
/** When the writer is called from the harness wrapping a child error. */
|
||||
errorClass?: string;
|
||||
errorCode?: string;
|
||||
@@ -141,6 +149,8 @@ export function logFriction(input: FrictionLogInput): void {
|
||||
if (input.agent) entry.agent = input.agent;
|
||||
if (input.transcriptOffset !== undefined) entry.transcript_offset = input.transcriptOffset;
|
||||
if (input.marker) entry.marker = input.marker;
|
||||
if (input.scenario) entry.scenario = input.scenario;
|
||||
if (input.harnessSchema !== undefined) entry.harness_schema = input.harnessSchema;
|
||||
|
||||
const line = JSON.stringify(entry) + '\n';
|
||||
appendFileSync(frictionFile(runId), line, 'utf-8');
|
||||
|
||||
@@ -434,6 +434,50 @@ export async function readSearchStats(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage disclosure for `readSearchStats()` consumers (`gbrain search
|
||||
* stats` / `gbrain search tune`). This documents the buffering behavior
|
||||
* from the module header above — it changes NO behavior, it only gives
|
||||
* display layers a single source of truth for the caveat text instead of
|
||||
* each caller re-describing (and risking drift on) the flush mechanics.
|
||||
*
|
||||
* Short-lived CLI invocations (a single `gbrain query "..."` call) usually
|
||||
* exit before the 60s timer or the 100-call threshold fires, so their
|
||||
* search is buffered in-memory and then lost with the process — never
|
||||
* written to `search_telemetry`. Long-lived processes (`gbrain serve`,
|
||||
* stdio/HTTP MCP, `gbrain jobs work`) survive long enough for the periodic
|
||||
* flush and are captured reliably. A CLI run that itself issues 100+
|
||||
* search calls before exiting (e.g. a bulk eval) CAN cross the threshold
|
||||
* and flush — hence "typically", not "never".
|
||||
*/
|
||||
export const TELEMETRY_COVERAGE_NOTE =
|
||||
'Counts are most complete for long-lived processes (gbrain serve, MCP stdio/HTTP, ' +
|
||||
'gbrain jobs work). A single short-lived CLI invocation typically exits before the ' +
|
||||
'telemetry buffer flushes (60s timer or 100-call threshold), so its search is ' +
|
||||
'usually not recorded here — see search/telemetry.ts for the buffering design.';
|
||||
|
||||
/**
|
||||
* Short, single-line form of {@link TELEMETRY_COVERAGE_NOTE} for human CLI
|
||||
* output (the long form is better suited to `--json`'s `reason` field).
|
||||
* Every human-facing caveat in `gbrain search stats`/`gbrain search tune`
|
||||
* reuses this literal string instead of paraphrasing it, so the wording
|
||||
* cannot drift between call sites.
|
||||
*/
|
||||
export const TELEMETRY_COVERAGE_CAVEAT =
|
||||
'Coverage favors long-lived processes (gbrain serve, MCP, jobs work) — a lone ' +
|
||||
'short-lived CLI search call is typically not recorded.';
|
||||
|
||||
export interface TelemetryCoverage {
|
||||
/** Whether a lone short-lived CLI search call is reliably counted. */
|
||||
cli_invocations: 'typically_not_recorded';
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** Machine-readable form of {@link TELEMETRY_COVERAGE_NOTE} for `--json` output. */
|
||||
export function telemetryCoverage(): TelemetryCoverage {
|
||||
return { cli_invocations: 'typically_not_recorded', reason: TELEMETRY_COVERAGE_NOTE };
|
||||
}
|
||||
|
||||
function nowDate(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# gbrain agent workspace — template
|
||||
|
||||
<!-- gbrain-template-stamp: 0.45.11.0 -->
|
||||
<!-- gbrain-template-stamp: 0.45.12.0 -->
|
||||
|
||||
This repository is the **"Use this template"** distribution artifact for a
|
||||
[gbrain](https://github.com/garrytan/gbrain) personal-agent workspace — the same
|
||||
|
||||
+241
-2
@@ -9,16 +9,17 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, existsSync, readFileSync } from 'fs';
|
||||
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, chmodSync, mkdirSync, symlinkSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { runFriction } from '../src/commands/friction.ts';
|
||||
import { listScenarios, loadScenario } from '../src/core/claw-test/scenarios.ts';
|
||||
import {
|
||||
registerAgentRunner, resolveAgentRunner, listRegisteredAgents,
|
||||
_resetRegistryForTests,
|
||||
_resetRegistryForTests, validateBinPathEnv,
|
||||
type AgentRunner, type DetectResult, type InvokeOpts, type InvokeResult,
|
||||
} from '../src/core/claw-test/agent-runner.ts';
|
||||
import { mergeChildFriction } from '../src/commands/claw-test.ts';
|
||||
|
||||
let tmp: string;
|
||||
const ORIG_HOME = process.env.GBRAIN_HOME;
|
||||
@@ -163,3 +164,241 @@ describe('OpenClawRunner detection (reliable on box without openclaw)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('HermesRunner detection (reliable on box without hermes)', () => {
|
||||
test('detect returns the contract shape when HERMES_BIN unset', async () => {
|
||||
const orig = process.env.HERMES_BIN;
|
||||
delete process.env.HERMES_BIN;
|
||||
try {
|
||||
const { HermesRunner } = await import('../src/core/claw-test/runners/hermes.ts');
|
||||
const d = await new HermesRunner().detect();
|
||||
// Available when hermes IS on the dev's PATH, unavailable otherwise —
|
||||
// both are valid; assert the contract shape only.
|
||||
expect(typeof d.available).toBe('boolean');
|
||||
if (!d.available) expect(typeof d.reason).toBe('string');
|
||||
else expect(d.binPath?.startsWith('/')).toBe(true);
|
||||
} finally {
|
||||
if (orig !== undefined) process.env.HERMES_BIN = orig;
|
||||
}
|
||||
});
|
||||
|
||||
test('detect rejects relative HERMES_BIN', async () => {
|
||||
const orig = process.env.HERMES_BIN;
|
||||
process.env.HERMES_BIN = 'relative/hermes';
|
||||
try {
|
||||
const { HermesRunner } = await import('../src/core/claw-test/runners/hermes.ts');
|
||||
const d = await new HermesRunner().detect();
|
||||
expect(d.available).toBe(false);
|
||||
expect(d.reason).toMatch(/HERMES_BIN must be absolute/);
|
||||
} finally {
|
||||
if (orig !== undefined) process.env.HERMES_BIN = orig;
|
||||
else delete process.env.HERMES_BIN;
|
||||
}
|
||||
});
|
||||
|
||||
test("detect rejects '..' segments in HERMES_BIN", async () => {
|
||||
const orig = process.env.HERMES_BIN;
|
||||
process.env.HERMES_BIN = '/tmp/foo/../hermes';
|
||||
try {
|
||||
const { HermesRunner } = await import('../src/core/claw-test/runners/hermes.ts');
|
||||
const d = await new HermesRunner().detect();
|
||||
expect(d.available).toBe(false);
|
||||
expect(d.reason).toMatch(/'\.\.' segments/);
|
||||
} finally {
|
||||
if (orig !== undefined) process.env.HERMES_BIN = orig;
|
||||
else delete process.env.HERMES_BIN;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('HermesRunner invoke argv/env (shim — no hermes binary needed)', () => {
|
||||
test('argv starts with the one-shot flag; HERMES_HOME propagates; unlisted env does not', async () => {
|
||||
const orig = {
|
||||
HERMES_BIN: process.env.HERMES_BIN,
|
||||
HERMES_HOME: process.env.HERMES_HOME,
|
||||
LEAK_CANARY: process.env.LEAK_CANARY,
|
||||
GBRAIN_DATABASE_URL: process.env.GBRAIN_DATABASE_URL,
|
||||
OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY,
|
||||
};
|
||||
const shim = join(tmp, 'hermes-shim');
|
||||
// Echo the argv and the env probes, then exit 0. The transcript sink
|
||||
// captures stdout, so assertions read the sink's events.
|
||||
writeFileSync(shim, '#!/bin/sh\nprintf "ARGV:%s\\n" "$@"\nprintf "HH:[%s] CANARY:[%s] DBURL:[%s] OR:[%s]\\n" "$HERMES_HOME" "$LEAK_CANARY" "$GBRAIN_DATABASE_URL" "$OPENROUTER_API_KEY"\n', 'utf-8');
|
||||
chmodSync(shim, 0o755);
|
||||
process.env.HERMES_BIN = shim;
|
||||
process.env.HERMES_HOME = '/tmp/hh-canary-test';
|
||||
process.env.LEAK_CANARY = 'must-not-leak';
|
||||
// Hermes-documented auth path (docs/mcp/HERMES.md): the hermes delta must
|
||||
// forward it or env-only OpenRouter operators get "no inference provider".
|
||||
process.env.OPENROUTER_API_KEY = 'or-sentinel-91c4';
|
||||
// Removed from BASE_ENV_ALLOWLIST in the adversarial review: an inherited
|
||||
// GBRAIN_DATABASE_URL would flip only the AGENT's gbrain to Postgres while
|
||||
// staging + the oracle stay on the hermetic PGLite (split-brain).
|
||||
process.env.GBRAIN_DATABASE_URL = 'postgres://must-not-leak';
|
||||
try {
|
||||
const { HermesRunner } = await import('../src/core/claw-test/runners/hermes.ts');
|
||||
const chunks: Buffer[] = [];
|
||||
const result = await new HermesRunner().invoke({
|
||||
cwd: tmp,
|
||||
brief: 'BRIEF BODY sentinel-7c2f',
|
||||
env: {},
|
||||
timeoutMs: 10_000,
|
||||
transcriptSink: {
|
||||
write: (e) => { if (e.channel === 'stdout') chunks.push(e.bytes); },
|
||||
nextOffset: () => 0,
|
||||
close: async () => {},
|
||||
},
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
const stdout = Buffer.concat(chunks).toString('utf-8');
|
||||
// First argv token is the one-shot flag, second is the brief itself.
|
||||
expect(stdout).toContain('ARGV:-z\nARGV:BRIEF BODY sentinel-7c2f');
|
||||
// Allowlist held: HERMES_HOME + OPENROUTER_API_KEY (the hermes delta)
|
||||
// pass; the canary and the deliberately-delisted GBRAIN_DATABASE_URL
|
||||
// don't.
|
||||
expect(stdout).toContain('HH:[/tmp/hh-canary-test]');
|
||||
expect(stdout).toContain('CANARY:[] DBURL:[] OR:[or-sentinel-91c4]');
|
||||
} finally {
|
||||
for (const [k, v] of Object.entries(orig)) {
|
||||
if (v !== undefined) process.env[k] = v;
|
||||
else delete process.env[k];
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenClawRunner invoke env (shim — pins the shared-allowlist leak barrier)', () => {
|
||||
test('GBRAIN_DATABASE_URL does not propagate through the openclaw runner either', async () => {
|
||||
// The split-brain fix removed GBRAIN_DATABASE_URL from BASE_ENV_ALLOWLIST;
|
||||
// the hermes shim test pins the hermes side — this pins the openclaw side
|
||||
// so an openclaw-specific delta re-adding it (the exact one-line
|
||||
// regression) cannot pass silently.
|
||||
const orig = { OPENCLAW_BIN: process.env.OPENCLAW_BIN, GBRAIN_DATABASE_URL: process.env.GBRAIN_DATABASE_URL };
|
||||
const shim = join(tmp, 'openclaw-shim');
|
||||
writeFileSync(shim, '#!/bin/sh\nprintf "DBURL:[%s]\\n" "$GBRAIN_DATABASE_URL"\n', 'utf-8');
|
||||
chmodSync(shim, 0o755);
|
||||
process.env.OPENCLAW_BIN = shim;
|
||||
process.env.GBRAIN_DATABASE_URL = 'postgres://must-not-leak';
|
||||
try {
|
||||
const { OpenClawRunner } = await import('../src/core/claw-test/runners/openclaw.ts');
|
||||
const chunks: Buffer[] = [];
|
||||
const result = await new OpenClawRunner().invoke({
|
||||
cwd: tmp,
|
||||
brief: 'brief',
|
||||
env: {},
|
||||
timeoutMs: 10_000,
|
||||
transcriptSink: {
|
||||
write: (e) => { if (e.channel === 'stdout') chunks.push(e.bytes); },
|
||||
nextOffset: () => 0,
|
||||
close: async () => {},
|
||||
},
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(Buffer.concat(chunks).toString('utf-8')).toContain('DBURL:[]');
|
||||
} finally {
|
||||
for (const [k, v] of Object.entries(orig)) {
|
||||
if (v !== undefined) process.env[k] = v;
|
||||
else delete process.env[k];
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateBinPathEnv — shim-quoting hardening', () => {
|
||||
test('rejects quote/metacharacter values that would break out of the generated shim quoting', () => {
|
||||
// The value is interpolated single-quoted into sh shim scripts; each of
|
||||
// these would otherwise become shell code.
|
||||
for (const bad of [
|
||||
"/tmp/x'; rm -rf /tmp/pwn; '",
|
||||
'/tmp/x"double',
|
||||
'/tmp/x`tick`',
|
||||
'/tmp/x$HOME',
|
||||
'/tmp/x\\backslash',
|
||||
'/tmp/x\nnewline',
|
||||
]) {
|
||||
expect(validateBinPathEnv('X_BIN', bad)).not.toBeNull();
|
||||
}
|
||||
// Spaces stay legal (macOS paths); quoting handles them.
|
||||
expect(validateBinPathEnv('X_BIN', '/Applications/App Support/gbrain')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeChildFriction — untrusted child file hardening', () => {
|
||||
// The child file lives in a workspace the AGENT writes to; the destination
|
||||
// is the operator's permanent friction log.
|
||||
const runId = 'claw-test-merge-hardening';
|
||||
|
||||
function childPath(runRoot: string): string {
|
||||
const dir = join(runRoot, '.gbrain', 'friction');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${runId}.jsonl`);
|
||||
}
|
||||
|
||||
function parentFile(): string {
|
||||
return join(tmp, '.gbrain', 'friction', `${runId}.jsonl`);
|
||||
}
|
||||
|
||||
test('valid JSONL lines merge; non-JSON and non-object lines are dropped', () => {
|
||||
const runRoot = join(tmp, 'runroot-valid');
|
||||
const entry = JSON.stringify({ phase: 'agent-side', message: 'kept', kind: 'friction' });
|
||||
writeFileSync(childPath(runRoot), `${entry}\nnot json at all\n"a json string scalar"\n[1,2]\n`, 'utf-8');
|
||||
mergeChildFriction(runRoot, runId);
|
||||
const merged = readFileSync(parentFile(), 'utf-8').split('\n').filter(l => l.trim());
|
||||
expect(merged).toEqual([entry]);
|
||||
});
|
||||
|
||||
test('a symlinked child file is refused (an agent-dropped link could import any readable file)', () => {
|
||||
const runRoot = join(tmp, 'runroot-symlink');
|
||||
const target = join(tmp, 'outside-secret.jsonl');
|
||||
writeFileSync(target, JSON.stringify({ phase: 'x', message: 'secret' }) + '\n', 'utf-8');
|
||||
const cp = childPath(runRoot);
|
||||
symlinkSync(target, cp);
|
||||
mergeChildFriction(runRoot, runId);
|
||||
expect(existsSync(parentFile())).toBe(false);
|
||||
});
|
||||
|
||||
test('an oversized child file is refused (size cap)', () => {
|
||||
const runRoot = join(tmp, 'runroot-huge');
|
||||
const line = JSON.stringify({ phase: 'x', message: 'y'.repeat(1024) });
|
||||
const lines = Math.ceil((5 * 1024 * 1024) / line.length) + 1;
|
||||
writeFileSync(childPath(runRoot), Array(lines).fill(line).join('\n') + '\n', 'utf-8');
|
||||
mergeChildFriction(runRoot, runId);
|
||||
expect(existsSync(parentFile())).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('spawnWithCapture — stdin EOF (no payload)', () => {
|
||||
test('an agent that waits for stdin EOF exits promptly instead of hanging to the timeout', async () => {
|
||||
const orig = process.env.HERMES_BIN;
|
||||
const shim = join(tmp, 'stdin-wait-shim');
|
||||
// `cat` blocks until stdin EOF; with stdin left open this burns the whole
|
||||
// timeout and exits 124 via the kill path.
|
||||
writeFileSync(shim, '#!/bin/sh\ncat > /dev/null\necho done\n', 'utf-8');
|
||||
chmodSync(shim, 0o755);
|
||||
process.env.HERMES_BIN = shim;
|
||||
try {
|
||||
const { HermesRunner } = await import('../src/core/claw-test/runners/hermes.ts');
|
||||
const start = Date.now();
|
||||
const result = await new HermesRunner().invoke({
|
||||
cwd: tmp,
|
||||
brief: 'brief',
|
||||
env: {},
|
||||
timeoutMs: 15_000,
|
||||
transcriptSink: { write: () => {}, nextOffset: () => 0, close: async () => {} },
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(Date.now() - start).toBeLessThan(10_000);
|
||||
} finally {
|
||||
if (orig !== undefined) process.env.HERMES_BIN = orig;
|
||||
else delete process.env.HERMES_BIN;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// NOTE deliberately absent: a "command module registers openclaw + hermes"
|
||||
// unit test. Any in-process version is a tautology — this file's beforeEach
|
||||
// wipes the registry and bun caches the command module, so the test would
|
||||
// have to re-register the runners itself and would pass even if the command
|
||||
// module dropped its registrations. The HONEST integration check lives in
|
||||
// test/e2e/claw-test.test.ts ("--list-agents reports both built-in runners"),
|
||||
// which spawns the real CLI and asserts both runner lines.
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runSearch } from '../src/commands/search.ts';
|
||||
import { recordSearchTelemetry, _resetTelemetryWriterForTest, getTelemetryWriter } from '../src/core/search/telemetry.ts';
|
||||
import { recordSearchTelemetry, _resetTelemetryWriterForTest, getTelemetryWriter, TELEMETRY_COVERAGE_CAVEAT } from '../src/core/search/telemetry.ts';
|
||||
import type { HybridSearchMeta } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
@@ -165,6 +165,81 @@ describe('gbrain search stats', () => {
|
||||
const outBig = await captureRun(() => runSearch(engine, ['stats', '--days', '9999', '--json']));
|
||||
expect(JSON.parse(outBig).window_days).toBe(365);
|
||||
});
|
||||
|
||||
// Coverage disclosure: short-lived CLI search calls typically don't
|
||||
// survive the telemetry flush timer/threshold, so `search stats` must
|
||||
// say so instead of presenting the (possibly CLI-blind) count as total.
|
||||
test('--json includes a coverage disclosure (empty table)', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['stats', '--json']));
|
||||
const stats = JSON.parse(out);
|
||||
expect(stats.coverage).toBeDefined();
|
||||
expect(stats.coverage.cli_invocations).toBe('typically_not_recorded');
|
||||
// Pin the substance, not just presence — an inaccurate reason string
|
||||
// (e.g. "long-lived processes only") must fail this test.
|
||||
expect(stats.coverage.reason).toMatch(/short-lived CLI/i);
|
||||
expect(stats.coverage.reason).toMatch(/typically.*not recorded|not.*typically recorded/i);
|
||||
});
|
||||
|
||||
test('--json includes a coverage disclosure (non-empty table)', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }), { results_count: 5 });
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['stats', '--json']));
|
||||
const stats = JSON.parse(out);
|
||||
expect(stats.coverage.cli_invocations).toBe('typically_not_recorded');
|
||||
});
|
||||
|
||||
// Wording-accuracy pin, independent of the TELEMETRY_COVERAGE_NOTE import:
|
||||
// importing the same constant into production code and the assertion
|
||||
// would let an inaccurate edit to the constant sail through unnoticed
|
||||
// (round-1 review caught exactly this class of bug — "long-lived
|
||||
// processes only" overclaimed and dropped `jobs work`). Hardcode the
|
||||
// substance here instead of comparing production output to itself.
|
||||
test('--json coverage.reason names all three long-lived process kinds + the threshold exception', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['stats', '--json']));
|
||||
const reason: string = JSON.parse(out).coverage.reason;
|
||||
expect(reason).toMatch(/gbrain serve/i);
|
||||
expect(reason).toMatch(/mcp/i);
|
||||
expect(reason).toMatch(/jobs work/i);
|
||||
expect(reason).toMatch(/short-lived CLI/i);
|
||||
// Must not claim CLI calls are NEVER recorded — a bulk CLI run that
|
||||
// itself crosses the 100-call flush threshold before exiting IS
|
||||
// captured, so the wording must hedge ("typically"/"usually"), not
|
||||
// assert absolute exclusivity ("only"/"never").
|
||||
expect(reason).toMatch(/typically|usually/i);
|
||||
expect(reason).not.toMatch(/\bonly\b/i);
|
||||
expect(reason).not.toMatch(/\bnever\b/i);
|
||||
});
|
||||
|
||||
test('human output surfaces the exact coverage caveat (empty table)', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['stats']));
|
||||
// Pin the literal shared constant — proves the display layer isn't
|
||||
// paraphrasing (and risking drift on) the buffering caveat.
|
||||
expect(out).toContain(TELEMETRY_COVERAGE_CAVEAT);
|
||||
expect(out.toLowerCase()).toContain('coverage gap above');
|
||||
});
|
||||
|
||||
// Same independent-wording-pin rationale as the --json test above,
|
||||
// applied to the short human caveat.
|
||||
test('human coverage caveat names long-lived processes + jobs work + the hedge word, independent of the import', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['stats']));
|
||||
expect(out).toMatch(/favors long-lived processes/i);
|
||||
expect(out).toMatch(/jobs work/i);
|
||||
expect(out).toMatch(/typically not recorded/i);
|
||||
expect(out).not.toMatch(/only long-lived processes/i);
|
||||
});
|
||||
|
||||
test('human output surfaces the exact coverage caveat (non-empty table)', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }), { results_count: 5 });
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['stats']));
|
||||
expect(out).toContain(TELEMETRY_COVERAGE_CAVEAT);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain search tune (recommendations)', () => {
|
||||
@@ -175,6 +250,26 @@ describe('gbrain search tune (recommendations)', () => {
|
||||
expect(r.recommendations).toEqual([]);
|
||||
});
|
||||
|
||||
// Coverage disclosure: `tune`'s recommendations are only as complete as
|
||||
// the telemetry they're read from — same caveat as `search stats`.
|
||||
test('insufficient data → --json includes coverage disclosure', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['tune', '--json']));
|
||||
const r = JSON.parse(out);
|
||||
expect(r.coverage).toBeDefined();
|
||||
expect(r.coverage.cli_invocations).toBe('typically_not_recorded');
|
||||
expect(r.coverage.reason).toMatch(/short-lived CLI/i);
|
||||
});
|
||||
|
||||
test('insufficient data → human output notes the exact coverage caveat', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['tune']));
|
||||
expect(out).toContain(TELEMETRY_COVERAGE_CAVEAT);
|
||||
// The old copy told the user to "run a few `gbrain query` calls" to fix
|
||||
// a zero count — that's misleading advice given the caveat (a single
|
||||
// CLI call is exactly what tends NOT to be recorded). Pin the corrected
|
||||
// suggestion instead.
|
||||
expect(out).toMatch(/gbrain serve.*or an MCP session/i);
|
||||
});
|
||||
|
||||
test('conservative + high budget drop rate → recommends balanced', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
const w = getTelemetryWriter();
|
||||
@@ -194,6 +289,25 @@ describe('gbrain search tune (recommendations)', () => {
|
||||
const modeRec = r.recommendations.find((x: { knob: string }) => x.knob === 'search.mode');
|
||||
expect(modeRec).toBeDefined();
|
||||
expect(modeRec.suggested).toBe('balanced');
|
||||
// Coverage disclosure travels with real recommendations too, not just
|
||||
// the insufficient-data early-return path.
|
||||
expect(r.coverage.cli_invocations).toBe('typically_not_recorded');
|
||||
});
|
||||
|
||||
test('has_recommendations → human output notes the exact coverage caveat', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
for (let i = 0; i < 30; i++) {
|
||||
recordSearchTelemetry(engine, makeMeta({
|
||||
mode: 'conservative',
|
||||
token_budget: { budget: 4000, used: 4000, kept: 5, dropped: 5 },
|
||||
}), { results_count: 5 });
|
||||
}
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['tune']));
|
||||
expect(out).toContain(TELEMETRY_COVERAGE_CAVEAT);
|
||||
});
|
||||
|
||||
test('tokenmax + Haiku subagent → recommends balanced', async () => {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* the real Anthropic call fails immediately, exhausting max_attempts and
|
||||
* landing the job in 'dead' (not 'timeout' — nothing ever times out, the
|
||||
* failure is immediate). The #2782 status-reflects-outcome contract this
|
||||
* test exists to pin is unchanged: any non-'complete' outcome with zero
|
||||
* test exists to pin is unchanged: any non-'completed' outcome with zero
|
||||
* writes must still surface as status 'fail', just under the outcome that
|
||||
* actually occurs now that the job is drained instead of left stuck in
|
||||
* 'waiting' for the full wait window.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
mock.module('../src/core/ai/gateway.ts', () => ({
|
||||
probeChatModel: () => ({ ok: true }),
|
||||
}));
|
||||
|
||||
mock.module('../src/core/cycle/synthesize.ts', () => ({
|
||||
loadAllowedSlugPrefixes: async () => ['wiki/personal/patterns/*'],
|
||||
loadOutputRoot: async () => 'wiki',
|
||||
runSubagentsInline: async () => undefined,
|
||||
}));
|
||||
|
||||
mock.module('../src/core/minions/wait-for-completion.ts', () => ({
|
||||
TimeoutError: class TimeoutError extends Error {},
|
||||
waitForCompletion: async (_queue: unknown, jobId: number) => ({
|
||||
id: jobId,
|
||||
status: 'completed',
|
||||
}),
|
||||
}));
|
||||
|
||||
const { runPhasePatterns } = await import('../src/core/cycle/patterns.ts');
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let schemaVersion: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
schemaVersion = (await engine.getConfig('version')) ?? '7';
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
await engine.setConfig('version', schemaVersion);
|
||||
await engine.setConfig('models.dream.patterns', 'anthropic:claude-sonnet-4-6');
|
||||
});
|
||||
|
||||
async function seedReflections(): Promise<void> {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO pages (slug, type, title, compiled_truth)
|
||||
VALUES ($1, 'note', $2, $3)`,
|
||||
[
|
||||
`wiki/personal/reflections/2026-08-0${i + 1}-reflection`,
|
||||
`Reflection ${i + 1}`,
|
||||
`Recurring theme fixture number ${i + 1}.`,
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
describe('runPhasePatterns completed child outcome (#4026)', () => {
|
||||
test('completed child with zero writes is an ok no-op, not PATTERNS_CHILD_COMPLETED', async () => {
|
||||
const brainDir = mkdtempSync(join(tmpdir(), 'gbrain-patterns-completed-'));
|
||||
try {
|
||||
await seedReflections();
|
||||
|
||||
const result = await runPhasePatterns(engine, { brainDir, dryRun: false });
|
||||
|
||||
expect(result.status).toBe('ok');
|
||||
expect(result.details.child_outcome).toBe('completed');
|
||||
expect(result.details.patterns_written).toBe(0);
|
||||
expect(result.error?.code).not.toBe('PATTERNS_CHILD_COMPLETED');
|
||||
} finally {
|
||||
rmSync(brainDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -71,6 +71,14 @@ describe('gbrain claw-test --scenario fresh-install (scripted)', () => {
|
||||
console.error('unexpected friction entries:', blockers);
|
||||
}
|
||||
expect(blockers.length).toBe(0);
|
||||
|
||||
// REGRESSION pin: scripted runs stamp agent 'scripted' (previously the
|
||||
// misleading parseArgs default 'openclaw'). friction diff's agent-name
|
||||
// resolution reads this stamp off the start marker.
|
||||
expect(entries[0].kind).toBe('phase-marker');
|
||||
expect(entries[0].marker).toBe('start');
|
||||
expect(entries[0].agent).toBe('scripted');
|
||||
expect(entries[0].scenario).toBe('fresh-install');
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
@@ -114,6 +122,292 @@ describe('gbrain claw-test --scenario fresh-install (scripted)', () => {
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
describe('gbrain claw-test --list-agents', () => {
|
||||
test('reports both built-in runners (available or not — both valid states)', () => {
|
||||
// HERMES_BIN/OPENCLAW_BIN point at a nonexistent path so the output shape
|
||||
// is deterministic regardless of what's installed on the box (detect
|
||||
// rejects a non-stat-able absolute path with a specific reason).
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--list-agents'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: { ...process.env, HERMES_BIN: '/nonexistent/hermes', OPENCLAW_BIN: '/nonexistent/openclaw' },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toMatch(/^hermes: unavailable: /m);
|
||||
expect(result.stdout).toMatch(/^openclaw: unavailable: /m);
|
||||
// Alphabetical print order (the awaited-detection fix pins this).
|
||||
expect(result.stdout.indexOf('hermes:')).toBeLessThan(result.stdout.indexOf('openclaw:'));
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('gbrain claw-test --live (shim agents; no real agent binary, no tokens)', () => {
|
||||
// The OpenClaw runner honors an absolute $OPENCLAW_BIN pointing at any
|
||||
// executable, so these tests drive live mode with tiny sh shims. That
|
||||
// exercises the REAL live path: staging, the PATH shim, the agent turn,
|
||||
// the success oracle, and the E0 friction merge.
|
||||
|
||||
function runLiveWithShim(shimBody: string, scenario: string, extraEnv: Record<string, string> = {}) {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-live-'));
|
||||
const shim = join(tmp, 'agent-shim');
|
||||
writeFileSync(shim, shimBody, 'utf-8');
|
||||
chmodSync(shim, 0o755);
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--live', '--agent', 'openclaw', '--scenario', scenario], {
|
||||
cwd: REPO_ROOT,
|
||||
env: {
|
||||
...process.env,
|
||||
GBRAIN_HOME: tmp,
|
||||
OPENCLAW_BIN: shim,
|
||||
GBRAIN_BIN_OVERRIDE: BIN_PATH,
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: extraEnv.GBRAIN_CLAW_SCENARIOS_DIR ?? SCENARIOS_DIR,
|
||||
...extraEnv,
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 180_000,
|
||||
});
|
||||
const frictionDirPath = join(tmp, '.gbrain', 'friction');
|
||||
const entries: any[] = [];
|
||||
if (existsSync(frictionDirPath)) {
|
||||
for (const f of readdirSync(frictionDirPath).filter(f => f.endsWith('.jsonl'))) {
|
||||
for (const line of readFileSync(join(frictionDirPath, f), 'utf-8').split('\n')) {
|
||||
if (line.trim()) entries.push(JSON.parse(line));
|
||||
}
|
||||
}
|
||||
}
|
||||
return { tmp, result, entries };
|
||||
}
|
||||
|
||||
test('oracle break path: a do-nothing agent that exits 0 now FAILS the run', () => {
|
||||
const { tmp, result, entries } = runLiveWithShim('#!/bin/sh\nexit 0\n', 'fresh-install');
|
||||
try {
|
||||
expect(result.status).not.toBe(0);
|
||||
const verifyErrors = entries.filter(e => e.phase === 'verify' && e.severity === 'error');
|
||||
expect(verifyErrors.length).toBeGreaterThan(0);
|
||||
// The run-start meta record (agent-name resolution depends on it) is the
|
||||
// FIRST line and carries agent + scenario.
|
||||
expect(entries[0].kind).toBe('phase-marker');
|
||||
expect(entries[0].marker).toBe('start');
|
||||
expect(entries[0].agent).toBe('openclaw');
|
||||
expect(entries[0].scenario).toBe('fresh-install');
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test('working agent passes the oracle AND its child-side friction survives tempdir cleanup (E0 merge)', () => {
|
||||
// The shim does the brief's work via BARE `gbrain` (proving the PATH shim
|
||||
// resolves to this checkout) and logs one agent-side friction entry, which
|
||||
// lands under the run's hermetic GBRAIN_HOME — the dir the harness deletes.
|
||||
const shim = [
|
||||
'#!/bin/sh',
|
||||
'set -e',
|
||||
'gbrain import ./brain --no-embed',
|
||||
'gbrain skillpack scaffold query --workspace "$PWD"',
|
||||
'gbrain friction log --phase agent-side --severity nit --message "child entry survives cleanup"',
|
||||
'',
|
||||
].join('\n');
|
||||
const { tmp, result, entries } = runLiveWithShim(shim, 'fresh-install');
|
||||
try {
|
||||
if (result.status !== 0) {
|
||||
console.error('STDOUT:', result.stdout);
|
||||
console.error('STDERR:', result.stderr);
|
||||
}
|
||||
expect(result.status).toBe(0);
|
||||
const blockers = entries.filter(e => e.kind === 'friction' && (e.severity === 'error' || e.severity === 'blocker'));
|
||||
expect(blockers.length).toBe(0);
|
||||
// E0: the child-side entry was written under the (deleted) runRoot but
|
||||
// must appear in the parent's friction file.
|
||||
const childSide = entries.filter(e => e.phase === 'agent-side' && e.message.includes('child entry survives cleanup'));
|
||||
expect(childSide.length).toBe(1);
|
||||
// Completion marker paired with the start marker.
|
||||
const endMarkers = entries.filter(e => e.kind === 'phase-marker' && e.marker === 'end' && e.phase === 'harness');
|
||||
expect(endMarkers.length).toBe(1);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test('REGRESSION (upgrade): staging seeds first and the schema-version oracle fails a do-nothing agent', () => {
|
||||
// Synthetic upgrade scenario: a minimal "old brain" dump that records
|
||||
// config.version = 1. The do-nothing agent never advances the migration
|
||||
// chain, so the non-mutating pre/post probe must fail the run. (A
|
||||
// doctor-based oracle would auto-migrate on connect and pass vacuously —
|
||||
// the exact bug class this oracle exists to prevent.)
|
||||
const scenRoot = mkdtempSync(join(tmpdir(), 'claw-test-e2e-upg-scen-'));
|
||||
const scenDir = join(scenRoot, 'upgrade-synthetic');
|
||||
mkdirSync(join(scenDir, 'seed'), { recursive: true });
|
||||
writeFileSync(join(scenDir, 'scenario.json'), JSON.stringify({
|
||||
kind: 'upgrade',
|
||||
from_version: '0.0.1',
|
||||
expected_phases: [],
|
||||
seed: 'seed',
|
||||
}), 'utf-8');
|
||||
writeFileSync(join(scenDir, 'BRIEF.md'), '# Upgrade brief\n\nRun `gbrain doctor --json` to walk the migration chain forward.\n', 'utf-8');
|
||||
writeFileSync(join(scenDir, 'seed', 'dump.sql'), [
|
||||
"CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT);",
|
||||
"INSERT INTO config (key, value) VALUES ('version', '1');",
|
||||
'',
|
||||
].join('\n'), 'utf-8');
|
||||
|
||||
const { tmp, result, entries } = runLiveWithShim('#!/bin/sh\nexit 0\n', 'upgrade-synthetic', {
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: scenRoot,
|
||||
});
|
||||
try {
|
||||
expect(result.status).not.toBe(0);
|
||||
const verifyErrors = entries.filter(e => e.phase === 'verify' && e.severity === 'error');
|
||||
expect(verifyErrors.length).toBe(1);
|
||||
expect(verifyErrors[0].message).toContain('did not advance');
|
||||
// pre=1 proves the seeded PGLite existed BEFORE the agent turn.
|
||||
expect(verifyErrors[0].message).toContain('pre=1');
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
rmSync(scenRoot, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('gbrain claw-test — adversarial-gate regression pins', () => {
|
||||
test('P1: without GBRAIN_BIN_OVERRIDE the harness synthesizes a bun launcher (children run gbrain, not bun)', () => {
|
||||
// Under `bun run src/cli.ts`, process.execPath is the Bun runtime; the
|
||||
// pre-fix fallback handed children `bun init …`, which scaffolds a Bun
|
||||
// project and fails the rest of the run. Exit 0 proves the synthesized
|
||||
// launcher resolved children to real gbrain.
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-noovr-'));
|
||||
try {
|
||||
const env: Record<string, string | undefined> = {
|
||||
...process.env,
|
||||
GBRAIN_HOME: tmp,
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: SCENARIOS_DIR,
|
||||
};
|
||||
delete env.GBRAIN_BIN_OVERRIDE;
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--scenario', 'fresh-install'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: env as NodeJS.ProcessEnv,
|
||||
encoding: 'utf-8',
|
||||
timeout: 150_000,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
console.error('STDOUT:', result.stdout);
|
||||
console.error('STDERR:', result.stderr);
|
||||
}
|
||||
expect(result.status).toBe(0);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test('scripted upgrade with the shipped fixture fails LOUDLY (no false-green upgrade without a seed dump)', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-upgloud-'));
|
||||
try {
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--scenario', 'upgrade-from-v0.18'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: {
|
||||
...process.env,
|
||||
GBRAIN_HOME: tmp,
|
||||
GBRAIN_BIN_OVERRIDE: BIN_PATH,
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: SCENARIOS_DIR,
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 60_000,
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('no seed dump');
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('charset guard: traversal-shaped scenario and agent values are usage errors, exit 2', () => {
|
||||
for (const argv of [
|
||||
['claw-test', '--scenario', '../evil'],
|
||||
['claw-test', '--agent', 'x/y', '--scenario', 'fresh-install'],
|
||||
]) {
|
||||
const result = spawnSync(BIN_PATH, argv, {
|
||||
cwd: REPO_ROOT,
|
||||
env: { ...process.env, GBRAIN_CLAW_SCENARIOS_DIR: SCENARIOS_DIR },
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status).toBe(2);
|
||||
expect(result.stderr).toContain('invalid');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test('phase timeout: a hung gbrain child is killed at GBRAIN_CLAW_PHASE_TIMEOUT_MS instead of wedging the run', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-hang-'));
|
||||
const hangBin = join(tmp, 'hang-gbrain');
|
||||
try {
|
||||
writeFileSync(hangBin, `#!/bin/sh\nif [ "$1" = "init" ]; then sleep 45; fi\nexec "${BIN_PATH}" "$@"\n`, 'utf-8');
|
||||
chmodSync(hangBin, 0o755);
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--scenario', 'fresh-install'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: {
|
||||
...process.env,
|
||||
GBRAIN_HOME: tmp,
|
||||
GBRAIN_BIN_OVERRIDE: hangBin,
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: SCENARIOS_DIR,
|
||||
GBRAIN_CLAW_PHASE_TIMEOUT_MS: '2000',
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 35_000,
|
||||
});
|
||||
// If the phase timeout were unwired, init would sleep past spawnSync's
|
||||
// own kill and status would be null with signal SIGTERM.
|
||||
expect(result.signal).toBeNull();
|
||||
expect(result.status).toBe(1);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test('fail-closed doctor oracle: doctor exiting 0 with unparsable output FAILS the live run', () => {
|
||||
// A wrapper gbrain that answers `doctor` with a non-JSON banner and exit 0
|
||||
// — the pre-fix oracle only rejected literal "unhealthy" and passed this.
|
||||
const scratch = mkdtempSync(join(tmpdir(), 'claw-test-e2e-docgarb-'));
|
||||
const wrapper = join(scratch, 'doctor-garbage-gbrain');
|
||||
writeFileSync(wrapper, `#!/bin/sh\nif [ "$1" = "doctor" ]; then echo "banner: everything is fine (not json)"; exit 0; fi\nexec "${BIN_PATH}" "$@"\n`, 'utf-8');
|
||||
chmodSync(wrapper, 0o755);
|
||||
const workShim = [
|
||||
'#!/bin/sh',
|
||||
'set -e',
|
||||
'gbrain import ./brain --no-embed',
|
||||
'gbrain skillpack scaffold query --workspace "$PWD"',
|
||||
'',
|
||||
].join('\n');
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-live-docgarb-'));
|
||||
const shim = join(tmp, 'agent-shim');
|
||||
writeFileSync(shim, workShim, 'utf-8');
|
||||
chmodSync(shim, 0o755);
|
||||
try {
|
||||
const result = spawnSync(BIN_PATH, ['claw-test', '--live', '--agent', 'openclaw', '--scenario', 'fresh-install'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: {
|
||||
...process.env,
|
||||
GBRAIN_HOME: tmp,
|
||||
OPENCLAW_BIN: shim,
|
||||
GBRAIN_BIN_OVERRIDE: wrapper,
|
||||
GBRAIN_CLAW_SCENARIOS_DIR: SCENARIOS_DIR,
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 180_000,
|
||||
});
|
||||
expect(result.status).not.toBe(0);
|
||||
const frictionDirPath = join(tmp, '.gbrain', 'friction');
|
||||
const entries: any[] = [];
|
||||
for (const f of readdirSync(frictionDirPath).filter(f => f.endsWith('.jsonl'))) {
|
||||
for (const line of readFileSync(join(frictionDirPath, f), 'utf-8').split('\n')) {
|
||||
if (line.trim()) entries.push(JSON.parse(line));
|
||||
}
|
||||
}
|
||||
const unparsable = entries.filter(e => e.phase === 'verify' && typeof e.message === 'string' && e.message.includes('unparsable'));
|
||||
expect(unparsable.length).toBe(1);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('gbrain friction render integration', () => {
|
||||
test('render produces a markdown report with the redact placeholder', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'claw-test-e2e-render-'));
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
/**
|
||||
* install-real-hermes door e2e — drives the REAL `hermes` binary (NousResearch
|
||||
* hermes-agent) against THIS checkout's gbrain over stdio MCP.
|
||||
*
|
||||
* WHAT THIS PROVES: gbrain WIRED INTO hermes + recall through MCP, using the
|
||||
* dev tree / compiled binary — the same posture as the claude/codex doors.
|
||||
* It does NOT prove (a) gbrain's own cold install (bun global install, PATH,
|
||||
* migrations — that's the networkless-container placeholder in
|
||||
* heavy-tests.yml), nor (b) an agent FOLLOWING INSTALL_FOR_AGENTS.md
|
||||
* end-to-end (guide-following is the claw-test live lane's job).
|
||||
*
|
||||
* GATING (fail-SKIP, never fail-hard): requires ALL of
|
||||
* - GBRAIN_REAL_HERMES_E2E=1 — explicit opt-in. This file matches the
|
||||
* ordinary test/e2e glob, and unlike the claude/codex doors (whose auth
|
||||
* probes die with a redirected HOME) an env-key auth gate alone would fire
|
||||
* paid multi-minute turns on any dev box with hermes + a key. CI's
|
||||
* hermes-door job and provisioned boxes set the var deliberately;
|
||||
* run-e2e.sh scrubs GBRAIN_* so this suite structurally cannot fire there.
|
||||
* - a resolvable hermes binary
|
||||
* - hermes auth with a NON-EMPTY provider key (blank CI secret ⇒ skip)
|
||||
*
|
||||
* Isolation: every child gets BOTH HOME=<tmp> and HERMES_HOME=<tmp>/.hermes
|
||||
* (HERMES_HOME honoring verified against v0.20.0; the double-set covers either
|
||||
* derivation). A tripwire hashes the operator's real ~/.hermes/config.yaml
|
||||
* before/after — if isolation ever breaks, the suite fails loudly instead of
|
||||
* silently mutating the operator's agent.
|
||||
*
|
||||
* Observed-reality notes (B0, docs/mcp/HERMES-CLI-PIN.md, v0.20.0):
|
||||
* - `hermes mcp add` does a REAL MCP handshake + tool discovery at add time,
|
||||
* then prompts to enable tools; a piped "Y" answers it non-interactively.
|
||||
* Its EXIT CODE IS 0 EVEN ON FAILURE/CANCEL — and a piped "Y" saves the
|
||||
* entry EVEN when the handshake failed (the save-anyway prompt), just with
|
||||
* `enabled: false`. So the hard success discriminators are
|
||||
* (1) `mcp_servers.gbrain.enabled === true` in config.yaml (only a
|
||||
* successful handshake enables) and (2) `hermes mcp test gbrain` exits 0.
|
||||
* - the args flag must be the LAST option (later options are swallowed into
|
||||
* the server argv).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { homedir, tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import * as yaml from 'js-yaml';
|
||||
|
||||
import {
|
||||
resolveHermesBinary,
|
||||
hasHermesAuth,
|
||||
seedHermesHome,
|
||||
pinHermesModel,
|
||||
hermesOneShotTurn,
|
||||
hermeticChildEnv,
|
||||
hermesChildEnv,
|
||||
resolveGbrainServerCommand,
|
||||
ensureCompiledGbrain,
|
||||
seedBrainForAgent,
|
||||
} from '../helpers/agent-harness.ts';
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..', '..');
|
||||
const CLI = join(REPO_ROOT, 'src', 'cli.ts');
|
||||
const HERMES_BIN = resolveHermesBinary();
|
||||
const CAN_RUN = process.env.GBRAIN_REAL_HERMES_E2E === '1' && !!HERMES_BIN && hasHermesAuth();
|
||||
|
||||
if (!CAN_RUN) {
|
||||
const why = process.env.GBRAIN_REAL_HERMES_E2E !== '1'
|
||||
? 'GBRAIN_REAL_HERMES_E2E is not 1 (explicit opt-in required — paid agent turns)'
|
||||
: !HERMES_BIN
|
||||
? 'hermes binary not found'
|
||||
: 'no non-empty hermes provider key (env or ~/.hermes/.env)';
|
||||
console.warn(`[install-real-hermes] SKIP: ${why}`);
|
||||
}
|
||||
|
||||
const ENV_KEYS = [
|
||||
'GBRAIN_HOME', 'GBRAIN_DATABASE_URL', 'DATABASE_URL', 'GBRAIN_BRAIN_ID',
|
||||
'GBRAIN_SOURCE', 'GBRAIN_HOOKS', 'HERMES_HOME',
|
||||
];
|
||||
const SAVED_ENV: Record<string, string | undefined> = {};
|
||||
|
||||
// Tripwire over the operator's REAL hermes config: hash before, compare after.
|
||||
const REAL_HERMES_CONFIG = join(homedir(), '.hermes', 'config.yaml');
|
||||
let realConfigHashBefore: string | null = null;
|
||||
function hashFile(p: string): string | null {
|
||||
try { return createHash('sha256').update(readFileSync(p)).digest('hex'); } catch { return null; }
|
||||
}
|
||||
|
||||
// Evidence trail: homes created during the run get copied (minus .env) into
|
||||
// GBRAIN_E2E_EVIDENCE_DIR when set. The CI workflow uploads that stable path
|
||||
// on failure; copying unconditionally is fine (upload is failure-gated) and
|
||||
// avoids per-test failure plumbing.
|
||||
const EVIDENCE_DIR = process.env.GBRAIN_E2E_EVIDENCE_DIR;
|
||||
const createdHomes: { label: string; home: string }[] = [];
|
||||
function trackHome(label: string): string {
|
||||
const home = mkdtempSync(join(tmpdir(), `gb-hermes-${label}-`));
|
||||
createdHomes.push({ label, home });
|
||||
return home;
|
||||
}
|
||||
function copyEvidence(): void {
|
||||
if (!EVIDENCE_DIR) return;
|
||||
for (const { label, home } of createdHomes) {
|
||||
try {
|
||||
const dst = join(EVIDENCE_DIR, label);
|
||||
mkdirSync(dst, { recursive: true });
|
||||
for (const sub of ['.hermes/logs', '.hermes/sessions', 'usage.json', 'door-config.yaml']) {
|
||||
const src = join(home, sub);
|
||||
if (existsSync(src)) {
|
||||
try { cpSync(src, join(dst, sub.replace(/\//g, '_')), { recursive: true }); } catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
// Defensive: never let a provider key land in the artifact.
|
||||
try { rmSync(join(dst, '.env'), { force: true }); } catch { /* best-effort */ }
|
||||
} catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
for (const k of ENV_KEYS) SAVED_ENV[k] = process.env[k];
|
||||
for (const k of ENV_KEYS) delete process.env[k];
|
||||
realConfigHashBefore = hashFile(REAL_HERMES_CONFIG);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
copyEvidence();
|
||||
for (const { home } of createdHomes) {
|
||||
try { rmSync(home, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
}
|
||||
for (const k of ENV_KEYS) {
|
||||
if (SAVED_ENV[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = SAVED_ENV[k];
|
||||
}
|
||||
// Tripwire LAST: if any child escaped the hermetic HERMES_HOME, the
|
||||
// operator's real config changed and this run must scream about it.
|
||||
const after = hashFile(REAL_HERMES_CONFIG);
|
||||
if (realConfigHashBefore !== after) {
|
||||
throw new Error(
|
||||
'HERMETICITY BREACH: the operator\'s real ~/.hermes/config.yaml changed during the door run — ' +
|
||||
'HERMES_HOME isolation failed; investigate before trusting this suite again.',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/** Run the real hermes binary under a hermetic home. `stdinText` answers
|
||||
* interactive prompts (the enable-tools "Y"). All provider keys are scrubbed
|
||||
* from the child env (hermesChildEnv) — the seeded .env is the single auth
|
||||
* source; a stray second provider key mis-routes hermes's provider-auto
|
||||
* (observed "HTTP 401: Missing Authentication header"). */
|
||||
function runHermes(home: string, argv: string[], stdinText?: string): { code: number | null; stdout: string; stderr: string } {
|
||||
const res = spawnSync(HERMES_BIN!, argv, {
|
||||
env: hermesChildEnv(home),
|
||||
encoding: 'utf8',
|
||||
timeout: 180_000,
|
||||
input: stdinText,
|
||||
});
|
||||
return { code: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '' };
|
||||
}
|
||||
|
||||
/** Keyless PGLite brain init via the preferred launcher (compiled binary when
|
||||
* the PGLite-embed probe passes, matching resolveGbrainServerCommand's
|
||||
* preference; bun-run fallback otherwise). */
|
||||
function initBrain(home: string): { code: number | null; stderr: string } {
|
||||
const { binPath } = ensureCompiledGbrain(REPO_ROOT);
|
||||
const argv = binPath
|
||||
? [binPath, 'init', '--pglite', '--no-embedding', '--non-interactive']
|
||||
: ['bun', 'run', CLI, 'init', '--pglite', '--no-embedding', '--non-interactive'];
|
||||
const res = spawnSync(argv[0], argv.slice(1), {
|
||||
cwd: REPO_ROOT,
|
||||
env: hermeticChildEnv({ HOME: home, GBRAIN_HOME: home, GBRAIN_SKIP_STARTUP_HOOKS: '1' }),
|
||||
encoding: 'utf8',
|
||||
timeout: 180_000,
|
||||
});
|
||||
return { code: res.status, stderr: `${res.stdout ?? ''}\n${res.stderr ?? ''}` };
|
||||
}
|
||||
|
||||
interface HermesMcpConfig {
|
||||
mcp_servers?: Record<string, { command?: string; args?: string[]; env?: Record<string, string>; enabled?: boolean }>;
|
||||
}
|
||||
|
||||
function readHermesConfig(home: string): HermesMcpConfig {
|
||||
const p = join(home, '.hermes', 'config.yaml');
|
||||
expect(existsSync(p)).toBe(true);
|
||||
return (yaml.safeLoad(readFileSync(p, 'utf-8')) ?? {}) as HermesMcpConfig;
|
||||
}
|
||||
|
||||
/** Shared registration: real `hermes mcp add` with the piped-Y confirmation.
|
||||
* Observed gotchas (door run + isolation, v0.20.0):
|
||||
* - the env flag takes MULTIPLE KEY=VALUE values after ONE flag; REPEATING
|
||||
* the flag replaces the first occurrence (argparse), which drops
|
||||
* GBRAIN_HOME, kills the server handshake, and the piped Y then answers
|
||||
* the save-anyway prompt as enabled:false — a silently-disabled entry.
|
||||
* - the args flag must be the LAST option (later options are swallowed
|
||||
* into the server argv). */
|
||||
function registerGbrainIntoHermes(
|
||||
home: string,
|
||||
server: { command: string; args: string[] },
|
||||
sourceId: string,
|
||||
): { code: number | null; stdout: string; stderr: string } {
|
||||
return runHermes(home, [
|
||||
'mcp', 'add', 'gbrain',
|
||||
'--env', `GBRAIN_HOME=${home}`, `GBRAIN_SOURCE=${sourceId}`,
|
||||
'--connect-timeout', '60',
|
||||
'--command', server.command,
|
||||
'--args', ...server.args,
|
||||
], 'Y\n');
|
||||
}
|
||||
|
||||
describe.skipIf(!CAN_RUN)('install real-hermes door (serial e2e)', () => {
|
||||
test('version pin: hermes --version matches HERMES_VERSION when the CI pin is set', () => {
|
||||
const pinned = SAVED_ENV.HERMES_VERSION ?? process.env.HERMES_VERSION;
|
||||
const res = runHermes(trackHome('ver'), ['--version']);
|
||||
expect(res.code).toBe(0);
|
||||
if (pinned) {
|
||||
expect(res.stdout).toContain(`v${pinned}`);
|
||||
} else {
|
||||
// Local run without a pin: still assert the observed output shape.
|
||||
expect(res.stdout).toMatch(/Hermes Agent v\d+\.\d+\.\d+/);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test('INSTALL: keyless init → real `hermes mcp add` handshake → config carries server + env → mcp test passes', () => {
|
||||
const home = trackHome('install');
|
||||
seedHermesHome(home);
|
||||
|
||||
const init = initBrain(home);
|
||||
expect(init.code).toBe(0);
|
||||
expect(existsSync(join(home, '.gbrain', 'brain.pglite'))).toBe(true);
|
||||
|
||||
const server = resolveGbrainServerCommand(REPO_ROOT);
|
||||
const add = registerGbrainIntoHermes(home, server, 'default');
|
||||
// Deliberately NO assertion on add.code: observed 0 even on failure — and
|
||||
// the piped Y saves even a FAILED handshake (as enabled:false via the
|
||||
// save-anyway prompt). The handshake success is asserted via
|
||||
// enabled===true below plus the independent mcp test probe…
|
||||
expect(add.stdout).toContain('tool(s)'); // discovery banner fired
|
||||
const cfg = readHermesConfig(home);
|
||||
try { cpSync(join(home, '.hermes', 'config.yaml'), join(home, 'door-config.yaml')); } catch { /* evidence */ }
|
||||
expect(cfg.mcp_servers?.gbrain).toBeDefined();
|
||||
expect(cfg.mcp_servers!.gbrain.command).toBe(server.command);
|
||||
expect(cfg.mcp_servers!.gbrain.args).toEqual(server.args);
|
||||
expect(cfg.mcp_servers!.gbrain.env?.GBRAIN_HOME).toBe(home);
|
||||
expect(cfg.mcp_servers!.gbrain.env?.GBRAIN_SOURCE).toBe('default');
|
||||
expect(cfg.mcp_servers!.gbrain.enabled).toBe(true);
|
||||
|
||||
// …and a second, independent connection: `hermes mcp test` re-spawns the
|
||||
// server and lists tools (observed exit 0 + tool list).
|
||||
const probe = runHermes(home, ['mcp', 'test', 'gbrain']);
|
||||
expect(probe.code).toBe(0);
|
||||
|
||||
// Soft probe (exit-0-only; output shape logged, not asserted).
|
||||
const list = runHermes(home, ['mcp', 'list']);
|
||||
expect(list.code).toBe(0);
|
||||
if (!list.stdout.includes('gbrain')) {
|
||||
console.warn('[install-real-hermes] mcp list output did not mention gbrain — shape drift? output:', list.stdout.slice(0, 400));
|
||||
}
|
||||
}, 300_000);
|
||||
|
||||
test('INSTALL 1b: the direct config.yaml surface (documented, not a fallback) is accepted independently', () => {
|
||||
const home = trackHome('yaml');
|
||||
seedHermesHome(home);
|
||||
|
||||
const init = initBrain(home);
|
||||
expect(init.code).toBe(0);
|
||||
|
||||
const server = resolveGbrainServerCommand(REPO_ROOT);
|
||||
const configPath = join(home, '.hermes', 'config.yaml');
|
||||
const doc = {
|
||||
mcp_servers: {
|
||||
gbrain: {
|
||||
command: server.command,
|
||||
args: server.args,
|
||||
env: { GBRAIN_HOME: home, GBRAIN_SOURCE: 'default' },
|
||||
connect_timeout: 60,
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
writeFileSync(configPath, yaml.safeDump(doc), 'utf-8');
|
||||
|
||||
// Targeted probe, asserted HARD (eng D3): `mcp test` connects to exactly
|
||||
// the entry under test. Global `doctor` is NOT asserted here — it
|
||||
// diagnoses hermes-wide health and can fail a hermetic home for reasons
|
||||
// unrelated to our entry; run it as logged evidence only.
|
||||
const probe = runHermes(home, ['mcp', 'test', 'gbrain']);
|
||||
expect(probe.code).toBe(0);
|
||||
expect(probe.stdout.length).toBeGreaterThan(0);
|
||||
|
||||
const doctor = runHermes(home, ['doctor']);
|
||||
console.warn(`[install-real-hermes] doctor (evidence only): exit ${doctor.code}`);
|
||||
}, 300_000);
|
||||
|
||||
test('SMOKE: real hermes -z answers the seeded fact through the gbrain MCP server', async () => {
|
||||
const home = trackHome('smoke');
|
||||
const seeded = await seedBrainForAgent(home, 'workspace');
|
||||
seedHermesHome(home);
|
||||
const pin = pinHermesModel(HERMES_BIN!, home);
|
||||
expect(pin.code).toBe(0);
|
||||
|
||||
const server = resolveGbrainServerCommand(REPO_ROOT);
|
||||
registerGbrainIntoHermes(home, server, 'workspace');
|
||||
const cfg = readHermesConfig(home);
|
||||
expect(cfg.mcp_servers?.gbrain?.enabled).toBe(true);
|
||||
|
||||
// Negative-control channel: hermes's one-shot prints only final text (no
|
||||
// tool-call stream to parse), so the prompt makes tool-absence loudly
|
||||
// detectable. The fact is 100% synthetic and the turn's cwd is the TEMP
|
||||
// HOME — never the repo checkout, where the committed fact in
|
||||
// agent-harness.ts would be greppable without MCP.
|
||||
const prompt =
|
||||
'You have an MCP server named gbrain connected to a knowledge brain. ' +
|
||||
`Using ONLY that brain (no general knowledge, no filesystem search), answer: ${seeded.query} ` +
|
||||
'Report exactly what the brain says. If no gbrain tool is available to you, reply with exactly: NO-GBRAIN-TOOL';
|
||||
|
||||
let finalText = '';
|
||||
let lastExit: number | null = null;
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
const turn = await hermesOneShotTurn({
|
||||
prompt,
|
||||
cwd: home,
|
||||
home,
|
||||
timeoutMs: 240_000,
|
||||
usageFile: join(home, 'usage.json'),
|
||||
});
|
||||
finalText = turn.finalText;
|
||||
lastExit = turn.exitCode;
|
||||
if (turn.exitCode === 0 && finalText.toLowerCase().includes('rivermouth')) break;
|
||||
console.warn(`[install-real-hermes] SMOKE attempt ${attempt}: exit=${turn.exitCode} text=${finalText.slice(0, 200)}`);
|
||||
if (attempt < 2) await new Promise((r) => setTimeout(r, 3_000));
|
||||
}
|
||||
|
||||
// Never-soften criteria: the synthetic fact surfaced AND the no-tool
|
||||
// control token did not.
|
||||
expect(lastExit).toBe(0);
|
||||
expect(finalText.toLowerCase()).toContain('rivermouth');
|
||||
expect(finalText).not.toContain('NO-GBRAIN-TOOL');
|
||||
|
||||
// Best-effort evidence sweep (logged, not asserted — first provisioned
|
||||
// run tells us whether session artifacts are promotable to hard asserts).
|
||||
try {
|
||||
const usage = JSON.parse(readFileSync(join(home, 'usage.json'), 'utf-8'));
|
||||
console.warn('[install-real-hermes] usage:', JSON.stringify(usage).slice(0, 300));
|
||||
} catch { /* absent — fine */ }
|
||||
}, 480_000);
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* test/e2e/workspace-generic-compat.test.ts — generic-workspace compat gate.
|
||||
*
|
||||
* Pins gbrain's DOCUMENTED CONTRACT for the INSTALL_FOR_AGENTS.md
|
||||
* "any repo with a workspace" flow: the cwd_walk_up detection tier
|
||||
* (src/core/repo-root.ts tier 1b), scaffold additivity
|
||||
* (src/core/skillpack/scaffold.ts contracts 1–3), and check-resolvable
|
||||
* against a root AGENTS.md with no manifest.json. Hermes is the
|
||||
* motivating consumer of this flow; the real Hermes-behavior proof is
|
||||
* the Phase B door test (test/e2e/install-real-hermes.serial.test.ts),
|
||||
* not this file — this one runs everywhere, PGLite/no-DB, ungated.
|
||||
*
|
||||
* Fixture: `test/fixtures/generic-agents-workspace/` — AGENTS.md at
|
||||
* workspace root, two skills below, deliberately nothing
|
||||
* OpenClaw-specific. Structural template: openclaw-reference-compat.test.ts.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, afterEach } from 'bun:test';
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync, appendFileSync, readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
import { checkResolvable } from '../../src/core/check-resolvable.ts';
|
||||
import { autoDetectSkillsDir } from '../../src/core/repo-root.ts';
|
||||
import { runScaffold } from '../../src/core/skillpack/scaffold.ts';
|
||||
import { findGbrainRoot } from '../../src/core/skillpack/bundle.ts';
|
||||
|
||||
const FIXTURE = join(import.meta.dir, '..', 'fixtures', 'generic-agents-workspace');
|
||||
const SKILLS_DIR = join(FIXTURE, 'skills');
|
||||
const REPO = join(import.meta.dir, '..', '..');
|
||||
const CLI = join(REPO, 'src', 'cli.ts');
|
||||
|
||||
const created: string[] = [];
|
||||
afterEach(() => {
|
||||
while (created.length) {
|
||||
const d = created.pop();
|
||||
if (d && existsSync(d)) rmSync(d, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe('generic agent-workspace compat (INSTALL_FOR_AGENTS.md flow)', () => {
|
||||
it('fixture shape: root AGENTS.md, skills/ below, no manifest.json', () => {
|
||||
expect(existsSync(FIXTURE)).toBe(true);
|
||||
expect(existsSync(join(FIXTURE, 'AGENTS.md'))).toBe(true);
|
||||
expect(existsSync(SKILLS_DIR)).toBe(true);
|
||||
expect(existsSync(join(SKILLS_DIR, 'manifest.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('auto-detects skills dir via the cwd_walk_up tier (no env needed)', () => {
|
||||
// IMPORTANT: the replacement env guards against the OPERATOR's
|
||||
// GBRAIN_SKILLS_DIR (tier 0) / OPENCLAW_WORKSPACE (tier 1) leaking
|
||||
// in from the test runner's environment; the home tier sits BELOW
|
||||
// cwd_walk_up in the priority order and could never preempt it.
|
||||
const emptyHome = mkdtempSync(join(tmpdir(), 'generic-ws-home-'));
|
||||
created.push(emptyHome);
|
||||
const detected = autoDetectSkillsDir(FIXTURE, { HOME: emptyHome });
|
||||
expect(detected.dir).toBe(SKILLS_DIR);
|
||||
expect(detected.source).toBe('cwd_walk_up');
|
||||
});
|
||||
|
||||
it('GBRAIN_SKILLS_DIR explicit override wins over cwd_walk_up', () => {
|
||||
// Tier 0 requires a resolver file inside the pointed-at dir, so the
|
||||
// override target ships its own AGENTS.md. Starting from FIXTURE
|
||||
// (which WOULD match cwd_walk_up), the explicit env must win.
|
||||
const emptyHome = mkdtempSync(join(tmpdir(), 'generic-ws-home-'));
|
||||
created.push(emptyHome);
|
||||
const override = mkdtempSync(join(tmpdir(), 'generic-ws-override-'));
|
||||
created.push(override);
|
||||
writeFileSync(join(override, 'AGENTS.md'), '# AGENTS\n');
|
||||
|
||||
const detected = autoDetectSkillsDir(FIXTURE, {
|
||||
GBRAIN_SKILLS_DIR: override,
|
||||
HOME: emptyHome,
|
||||
});
|
||||
expect(detected.dir).toBe(override);
|
||||
expect(detected.source).toBe('env_explicit');
|
||||
});
|
||||
|
||||
it('checkResolvable accepts root AGENTS.md — all skills reachable, no errors', () => {
|
||||
const report = checkResolvable(SKILLS_DIR);
|
||||
expect(report.ok).toBe(true);
|
||||
expect(report.errors).toEqual([]);
|
||||
expect(report.summary.total_skills).toBe(2);
|
||||
expect(report.summary.reachable).toBe(2);
|
||||
expect(report.summary.unreachable).toBe(0);
|
||||
});
|
||||
|
||||
it('CLI subprocess: check-resolvable with json + skills-dir flags exits 0, JSON clean', () => {
|
||||
const r = spawnSync(
|
||||
'bun',
|
||||
[CLI, 'check-resolvable', '--json', '--skills-dir', SKILLS_DIR],
|
||||
{ encoding: 'utf-8', cwd: REPO, maxBuffer: 10 * 1024 * 1024 },
|
||||
);
|
||||
expect(r.status).toBe(0);
|
||||
const env = JSON.parse(r.stdout);
|
||||
expect(env.ok).toBe(true);
|
||||
expect(env.report.errors).toEqual([]);
|
||||
expect(env.report.summary.total_skills).toBe(2);
|
||||
});
|
||||
|
||||
it('scaffold is additive into an AGENTS.md-shell workspace and refuses overwrite on re-run', () => {
|
||||
// Fresh workspace with only the AGENTS.md shell — the documented
|
||||
// starting state for a generic repo adopting gbrain skills.
|
||||
const target = mkdtempSync(join(tmpdir(), 'generic-ws-scaffold-'));
|
||||
created.push(target);
|
||||
const shell = '# AGENTS\n\n| Trigger | Skill |\n|---------|-------|\n';
|
||||
writeFileSync(join(target, 'AGENTS.md'), shell);
|
||||
|
||||
const gbrainRoot = findGbrainRoot();
|
||||
expect(gbrainRoot).not.toBeNull();
|
||||
|
||||
const first = runScaffold({
|
||||
gbrainRoot: gbrainRoot!,
|
||||
targetWorkspace: target,
|
||||
skillSlug: 'query',
|
||||
});
|
||||
expect(first.summary.wroteNew).toBeGreaterThan(0);
|
||||
// The skill's SKILL.md lands under skills/query/; every file the
|
||||
// scaffold reports as written actually exists; paired sources (from
|
||||
// frontmatter, when the skill declares any) are written, not skipped.
|
||||
const skillMd = join(target, 'skills', 'query', 'SKILL.md');
|
||||
expect(existsSync(skillMd)).toBe(true);
|
||||
for (const f of first.files) {
|
||||
if (f.outcome === 'wrote_new') expect(existsSync(f.target)).toBe(true);
|
||||
}
|
||||
expect(first.summary.pairedSourcesWritten).toBe(
|
||||
first.files.filter(f => f.outcome === 'wrote_new' && f.pairedSource).length,
|
||||
);
|
||||
|
||||
// check-resolvable stays clean after scaffold: the scaffolded skill
|
||||
// is reachable via its own frontmatter triggers (scaffold contract 1
|
||||
// — no managed-block writes needed).
|
||||
const report = checkResolvable(join(target, 'skills'));
|
||||
expect(report.ok).toBe(true);
|
||||
expect(report.errors).toEqual([]);
|
||||
expect(report.summary.unreachable).toBe(0);
|
||||
|
||||
// Contract 2 (scaffold.ts header): once a file lands, the user owns
|
||||
// it — a re-run must skip every existing file and preserve edits.
|
||||
const marker = '\n<!-- user-owned edit: alice-example -->\n';
|
||||
appendFileSync(skillMd, marker);
|
||||
const second = runScaffold({
|
||||
gbrainRoot: gbrainRoot!,
|
||||
targetWorkspace: target,
|
||||
skillSlug: 'query',
|
||||
});
|
||||
expect(second.summary.wroteNew).toBe(0);
|
||||
expect(second.summary.skippedExisting).toBe(first.summary.wroteNew);
|
||||
expect(readFileSync(skillMd, 'utf-8')).toContain(marker.trim());
|
||||
// Byproduct check: the host resolver file was never touched.
|
||||
expect(readFileSync(join(target, 'AGENTS.md'), 'utf-8')).toBe(shell);
|
||||
});
|
||||
});
|
||||
@@ -3,11 +3,11 @@
|
||||
You are testing gbrain on a brand-new install. The user just ran `gbrain init` for the first time. Walk through the canonical first-day flow:
|
||||
|
||||
1. **Verify install:** confirm `gbrain --version` works and `gbrain doctor --json` returns a valid JSON object with a `status` field.
|
||||
2. **Install skillpack:** run `gbrain skillpack install --workspace $PWD`. The workspace already has an `AGENTS.md` routing file.
|
||||
2. **Scaffold the skillpack:** run `gbrain skillpack scaffold --all --workspace $PWD`. The workspace already has an `AGENTS.md` routing file.
|
||||
3. **Import the brain:** run `gbrain import ./brain --no-embed --progress-json`. There are 3 small markdown pages already there.
|
||||
4. **Query the brain:** run `gbrain query "alice"` and verify >0 results.
|
||||
5. **Extract links:** run `gbrain extract --source fs --progress-json`.
|
||||
6. **Verify health:** run `gbrain doctor --json`. The `status` field should be `"ok"`.
|
||||
5. **Extract links:** run `gbrain extract all --source fs --dir ./brain --progress-json`.
|
||||
6. **Verify health:** run `gbrain doctor --json`. The `status` field should be `"healthy"` or `"warnings"` — never `"unhealthy"`.
|
||||
|
||||
## Friction protocol
|
||||
|
||||
|
||||
@@ -6,5 +6,10 @@
|
||||
"extract.links_fs",
|
||||
"doctor.db_checks"
|
||||
],
|
||||
"brain": "brain"
|
||||
"brain": "brain",
|
||||
"oracle": {
|
||||
"query": "alice",
|
||||
"min_results": 1,
|
||||
"files_exist": ["skills/query/SKILL.md"]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ You inherit a gbrain v0.18 brain (the harness has already replayed a seed SQL du
|
||||
|
||||
1. **Run `gbrain doctor --json`** first. Note any warnings or fix-hints.
|
||||
2. **Run `gbrain init --pglite`** with the existing database path. The migration chain should detect the old `schema_version` and walk forward to the latest.
|
||||
3. **Run `gbrain doctor --json` again.** The `status` field should be `"ok"`.
|
||||
3. **Run `gbrain doctor --json` again.** The `status` field should be `"healthy"` or `"warnings"` — never `"unhealthy"`.
|
||||
4. **Verify queries still work:** `gbrain query "alice"` should return results from the seeded brain.
|
||||
|
||||
## Friction protocol
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"min_pages_after_migration": 1,
|
||||
"doctor_status": "ok"
|
||||
}
|
||||
@@ -6,5 +6,9 @@
|
||||
"doctor.db_checks"
|
||||
],
|
||||
"seed": "seed",
|
||||
"brain": "brain"
|
||||
"brain": "brain",
|
||||
"oracle": {
|
||||
"query": "alice",
|
||||
"min_results": 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# AGENTS.md
|
||||
|
||||
Minimal fixture mimicking a GENERIC agent workspace — the shape
|
||||
INSTALL_FOR_AGENTS.md's "any repo with a workspace" flow targets.
|
||||
AGENTS.md lives at workspace root; skills live under `skills/`. No
|
||||
manifest.json (the auto-derive path in `src/core/skill-manifest.ts`
|
||||
handles this). Unlike `openclaw-reference-minimal/`, nothing here is
|
||||
OpenClaw-specific: no OPENCLAW_WORKSPACE env, no plugin layout — just
|
||||
a repo with a root AGENTS.md and a bare `skills/` directory. All
|
||||
content is synthetic (alice-example style placeholders only).
|
||||
|
||||
## Brain operations
|
||||
|
||||
| Trigger | Skill |
|
||||
|---------|-------|
|
||||
| "what do we know about", "search for", "lookup" | `skills/query/SKILL.md` |
|
||||
| any brain read/write/lookup/citation | `skills/brain-ops/SKILL.md` |
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
name: brain-ops
|
||||
description: Core read/write cycle for the generic agent-workspace fixture.
|
||||
triggers:
|
||||
- any brain read/write/lookup/citation
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- people/
|
||||
- companies/
|
||||
---
|
||||
|
||||
# brain-ops
|
||||
|
||||
Fixture skill for `test/e2e/workspace-generic-compat.test.ts`.
|
||||
Example filing targets: `people/alice-example.md`, `companies/acme-example.md`.
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
name: query
|
||||
description: Look up brain pages in the generic agent-workspace fixture.
|
||||
triggers:
|
||||
- "what do we know about"
|
||||
- "search for"
|
||||
- "lookup"
|
||||
---
|
||||
|
||||
# query
|
||||
|
||||
Fixture skill for `test/e2e/workspace-generic-compat.test.ts`.
|
||||
Example lookup: "what do we know about alice-example".
|
||||
@@ -0,0 +1,378 @@
|
||||
/**
|
||||
* gbrain friction diff — cross-run comparison tests (Phase E).
|
||||
*
|
||||
* Zero-binary: exercises runFriction dispatch + the exported diff helpers
|
||||
* directly against a tmp GBRAIN_HOME, same conventions as friction-cli.test.ts.
|
||||
*
|
||||
* Spec anchors (plan Phase E + adversarial gate): identity is (kind, phase,
|
||||
* normalized message prefix — digits collapsed) with severity EXCLUDED (it is
|
||||
* the compared attribute, as a per-severity multiset); identities are
|
||||
* MULTISETS (count deltas are differences); only kind friction|delight is
|
||||
* diffable (markers/interruptions feed the compatibility banner); exit codes
|
||||
* 0 = ran, 1 = resolution/IO error, 2 = usage error.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, appendFileSync, utimesSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { runFriction, computeFrictionDiff, resolveRunSpec } from '../src/commands/friction.ts';
|
||||
import { logFriction, frictionFile } from '../src/core/friction.ts';
|
||||
import type { FrictionSeverity } from '../src/core/friction.ts';
|
||||
|
||||
const ORIG_HOME = process.env.GBRAIN_HOME;
|
||||
const ORIG_RUN_ID = process.env.GBRAIN_FRICTION_RUN_ID;
|
||||
let tmp: string;
|
||||
let stdoutLines: string[];
|
||||
let stderrLines: string[];
|
||||
let origStdoutWrite: typeof process.stdout.write;
|
||||
let origConsoleLog: typeof console.log;
|
||||
let origConsoleError: typeof console.error;
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = mkdtempSync(join(tmpdir(), 'friction-diff-'));
|
||||
process.env.GBRAIN_HOME = tmp;
|
||||
delete process.env.GBRAIN_FRICTION_RUN_ID;
|
||||
stdoutLines = [];
|
||||
stderrLines = [];
|
||||
origStdoutWrite = process.stdout.write.bind(process.stdout);
|
||||
origConsoleLog = console.log;
|
||||
origConsoleError = console.error;
|
||||
process.stdout.write = ((chunk: string) => { stdoutLines.push(String(chunk)); return true; }) as any;
|
||||
console.log = (...args: unknown[]) => { stdoutLines.push(args.join(' ') + '\n'); };
|
||||
console.error = (...args: unknown[]) => { stderrLines.push(args.join(' ') + '\n'); };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.GBRAIN_HOME = ORIG_HOME;
|
||||
if (ORIG_RUN_ID !== undefined) process.env.GBRAIN_FRICTION_RUN_ID = ORIG_RUN_ID;
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
process.stdout.write = origStdoutWrite;
|
||||
console.log = origConsoleLog;
|
||||
console.error = origConsoleError;
|
||||
});
|
||||
|
||||
/** Stamp the run-start marker the harness writes on every run. */
|
||||
function startMarker(runId: string, opts: { agent?: string; scenario?: string } = {}): void {
|
||||
logFriction({
|
||||
runId, phase: 'run', kind: 'phase-marker', marker: 'start',
|
||||
message: 'run start', source: 'harness',
|
||||
agent: opts.agent, scenario: opts.scenario,
|
||||
});
|
||||
}
|
||||
|
||||
function friction(runId: string, phase: string, message: string, severity: FrictionSeverity = 'error'): void {
|
||||
logFriction({ runId, phase, message, severity, kind: 'friction', source: 'claw' });
|
||||
}
|
||||
|
||||
function diffJson(base: string, compare: string): { code: number; parsed: any } {
|
||||
stdoutLines.length = 0;
|
||||
const code = runFriction(['diff', '--base', base, '--compare', compare, '--json']);
|
||||
const out = stdoutLines.join('').trim();
|
||||
return { code, parsed: out ? JSON.parse(out) : undefined };
|
||||
}
|
||||
|
||||
describe('three sections', () => {
|
||||
test('unique-to-compare, unique-to-base, and severity change land in the right buckets', () => {
|
||||
startMarker('run-base', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
friction('run-base', 'install', 'init did not say which engine', 'error');
|
||||
startMarker('run-cmp', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
friction('run-cmp', 'install', 'init did not say which engine', 'blocker'); // X, severity bumped
|
||||
friction('run-cmp', 'query', 'query returned nothing for a seeded doc', 'confused'); // Y
|
||||
|
||||
const { code, parsed } = diffJson('run-base', 'run-cmp');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.unique_to_base).toEqual([]);
|
||||
expect(parsed.unique_to_compare.length).toBe(1);
|
||||
expect(parsed.unique_to_compare[0].phase).toBe('query');
|
||||
expect(parsed.unique_to_compare[0].message).toContain('query returned nothing');
|
||||
expect(parsed.changed.length).toBe(1);
|
||||
expect(parsed.changed[0].phase).toBe('install');
|
||||
expect(parsed.changed[0].severity_changed).toBe(true);
|
||||
expect(parsed.changed[0].count_changed).toBe(false);
|
||||
expect(parsed.changed[0].base_severities).toEqual(['error']);
|
||||
expect(parsed.changed[0].compare_severities).toEqual(['blocker']);
|
||||
});
|
||||
|
||||
test('human output labels sections by run-id (instrument, not judge) and groups by phase', () => {
|
||||
startMarker('run-base', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
friction('run-base', 'install', 'only in base run');
|
||||
startMarker('run-cmp', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
friction('run-cmp', 'query', 'only in compare run');
|
||||
|
||||
stdoutLines.length = 0;
|
||||
const code = runFriction(['diff', '--base', 'run-base', '--compare', 'run-cmp']);
|
||||
expect(code).toBe(0);
|
||||
const out = stdoutLines.join('');
|
||||
expect(out).toContain('Unique to `run-cmp`');
|
||||
expect(out).toContain('Unique to `run-base`');
|
||||
expect(out).toContain('Shared but changed');
|
||||
expect(out).toContain('### `install`');
|
||||
expect(out).toContain('### `query`');
|
||||
});
|
||||
});
|
||||
|
||||
describe('multiset counts', () => {
|
||||
test('1×error vs 10×error of the same identity is a count delta, not equality', () => {
|
||||
startMarker('run-a', { agent: 'scripted' });
|
||||
friction('run-a', 'embed', 'embed batch rejected by provider', 'error');
|
||||
startMarker('run-b', { agent: 'scripted' });
|
||||
for (let i = 0; i < 10; i++) friction('run-b', 'embed', 'embed batch rejected by provider', 'error');
|
||||
|
||||
const { code, parsed } = diffJson('run-a', 'run-b');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.unique_to_base).toEqual([]);
|
||||
expect(parsed.unique_to_compare).toEqual([]);
|
||||
expect(parsed.changed.length).toBe(1);
|
||||
expect(parsed.changed[0].base_count).toBe(1);
|
||||
expect(parsed.changed[0].compare_count).toBe(10);
|
||||
expect(parsed.changed[0].count_changed).toBe(true);
|
||||
expect(parsed.changed[0].severity_changed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('kind filter', () => {
|
||||
test('phase-marker and interrupted entries never enter the three sections; they feed the banner', () => {
|
||||
startMarker('run-a', { agent: 'scripted' });
|
||||
logFriction({ runId: 'run-a', phase: 'run', kind: 'phase-marker', marker: 'end', message: 'MARKER SENTINEL A', source: 'harness' });
|
||||
logFriction({ runId: 'run-a', phase: 'run', kind: 'interrupted', message: 'INTERRUPT SENTINEL A', source: 'harness' });
|
||||
friction('run-a', 'install', 'real base friction');
|
||||
startMarker('run-b', { agent: 'scripted' });
|
||||
logFriction({ runId: 'run-b', phase: 'run', kind: 'phase-marker', marker: 'end', message: 'MARKER SENTINEL B', source: 'harness' });
|
||||
logFriction({ runId: 'run-b', phase: 'run', kind: 'interrupted', message: 'INTERRUPT SENTINEL B', source: 'harness' });
|
||||
friction('run-b', 'install', 'real compare friction');
|
||||
|
||||
const { code, parsed } = diffJson('run-a', 'run-b');
|
||||
expect(code).toBe(0);
|
||||
const sections = JSON.stringify([parsed.unique_to_base, parsed.unique_to_compare, parsed.changed]);
|
||||
expect(sections).not.toContain('SENTINEL');
|
||||
expect(parsed.unique_to_base.length).toBe(1);
|
||||
expect(parsed.unique_to_compare.length).toBe(1);
|
||||
expect(parsed.banner.base.interrupted).toBe(true);
|
||||
expect(parsed.banner.compare.interrupted).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('run resolution', () => {
|
||||
test('agent name resolves to the LATEST run carrying that agent', () => {
|
||||
startMarker('older-run', { agent: 'hermes' });
|
||||
friction('older-run', 'install', 'old friction');
|
||||
startMarker('newer-run', { agent: 'hermes' });
|
||||
friction('newer-run', 'install', 'new friction');
|
||||
// Deterministic mtimes: older-run well in the past, newer-run now.
|
||||
const past = new Date(Date.now() - 60_000);
|
||||
utimesSync(frictionFile('older-run'), past, past);
|
||||
|
||||
expect(resolveRunSpec('hermes')).toBe('newer-run');
|
||||
|
||||
const { code, parsed } = diffJson('hermes', 'older-run');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.base).toBe('newer-run');
|
||||
expect(parsed.compare).toBe('older-run');
|
||||
});
|
||||
|
||||
test('an exact run-id wins over agent-name interpretation', () => {
|
||||
// A run whose run-id IS the string 'hermes', carrying a different agent...
|
||||
startMarker('hermes', { agent: 'scripted' });
|
||||
// ...and a newer run whose AGENT is 'hermes'.
|
||||
startMarker('agent-stamped-run', { agent: 'hermes' });
|
||||
const past = new Date(Date.now() - 60_000);
|
||||
utimesSync(frictionFile('hermes'), past, past);
|
||||
|
||||
expect(resolveRunSpec('hermes')).toBe('hermes');
|
||||
});
|
||||
});
|
||||
|
||||
describe('failure semantics', () => {
|
||||
test('unknown run or agent exits 1 and stderr lists available runs', () => {
|
||||
startMarker('run-known', { agent: 'scripted' });
|
||||
stderrLines.length = 0;
|
||||
const code = runFriction(['diff', '--base', 'no-such-run', '--compare', 'run-known']);
|
||||
expect(code).toBe(1);
|
||||
const err = stderrLines.join('');
|
||||
expect(err).toContain('no-such-run');
|
||||
expect(err).toContain('available runs');
|
||||
expect(err).toContain('run-known');
|
||||
});
|
||||
|
||||
test('no runs at all: unknown spec exits 1 with an empty available list', () => {
|
||||
const code = runFriction(['diff', '--base', 'ghost-a', '--compare', 'ghost-b']);
|
||||
expect(code).toBe(1);
|
||||
expect(stderrLines.join('')).toContain('(none)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('empty and identical runs', () => {
|
||||
test('a run with only a start marker produces a valid empty diff, exit 0', () => {
|
||||
startMarker('empty-a', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
startMarker('empty-b', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
const { code, parsed } = diffJson('empty-a', 'empty-b');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.unique_to_base).toEqual([]);
|
||||
expect(parsed.unique_to_compare).toEqual([]);
|
||||
expect(parsed.changed).toEqual([]);
|
||||
expect(parsed.banner.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
test('base equals compare: valid no-difference output, exit 0', () => {
|
||||
startMarker('run-same', { agent: 'scripted' });
|
||||
friction('run-same', 'install', 'a friction entry');
|
||||
const { code, parsed } = diffJson('run-same', 'run-same');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.base).toBe('run-same');
|
||||
expect(parsed.compare).toBe('run-same');
|
||||
expect(parsed.unique_to_base).toEqual([]);
|
||||
expect(parsed.unique_to_compare).toEqual([]);
|
||||
expect(parsed.changed).toEqual([]);
|
||||
|
||||
stdoutLines.length = 0;
|
||||
expect(runFriction(['diff', '--base', 'run-same', '--compare', 'run-same'])).toBe(0);
|
||||
expect(stdoutLines.join('')).toContain('No differences');
|
||||
});
|
||||
});
|
||||
|
||||
describe('malformed JSONL tolerance', () => {
|
||||
test('a malformed line is skipped, its count surfaces, and the diff is still produced', () => {
|
||||
startMarker('mal-run', { agent: 'scripted' });
|
||||
appendFileSync(frictionFile('mal-run'), '{this is not json\n', 'utf-8');
|
||||
friction('mal-run', 'install', 'friction after the bad line');
|
||||
startMarker('clean-run', { agent: 'scripted' });
|
||||
|
||||
const { code, parsed } = diffJson('mal-run', 'clean-run');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.banner.base.malformed).toBe(1);
|
||||
expect(parsed.banner.compare.malformed).toBe(0);
|
||||
expect(parsed.unique_to_base.length).toBe(1);
|
||||
|
||||
stdoutLines.length = 0;
|
||||
runFriction(['diff', '--base', 'mal-run', '--compare', 'clean-run']);
|
||||
expect(stdoutLines.join('')).toContain('1 malformed line(s) skipped');
|
||||
});
|
||||
});
|
||||
|
||||
describe('compatibility banner', () => {
|
||||
test('scenario mismatch warns loudly, naming both scenarios', () => {
|
||||
startMarker('scen-a', { agent: 'scripted', scenario: 'fresh-install' });
|
||||
startMarker('scen-b', { agent: 'scripted', scenario: 'upgrade-from-v0.18' });
|
||||
|
||||
const { code, parsed } = diffJson('scen-a', 'scen-b');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.banner.warnings.length).toBeGreaterThanOrEqual(1);
|
||||
const joined = parsed.banner.warnings.join('\n');
|
||||
expect(joined).toContain('fresh-install');
|
||||
expect(joined).toContain('upgrade-from-v0.18');
|
||||
|
||||
stdoutLines.length = 0;
|
||||
runFriction(['diff', '--base', 'scen-a', '--compare', 'scen-b']);
|
||||
const out = stdoutLines.join('');
|
||||
expect(out).toContain('WARN');
|
||||
expect(out).toContain('fresh-install');
|
||||
expect(out).toContain('upgrade-from-v0.18');
|
||||
});
|
||||
|
||||
test('banner carries agent, scenario, and gbrain version from the start markers', () => {
|
||||
startMarker('meta-a', { agent: 'hermes', scenario: 'fresh-install' });
|
||||
startMarker('meta-b', { agent: 'openclaw', scenario: 'fresh-install' });
|
||||
const { parsed } = diffJson('meta-a', 'meta-b');
|
||||
expect(parsed.banner.base.agent).toBe('hermes');
|
||||
expect(parsed.banner.compare.agent).toBe('openclaw');
|
||||
expect(parsed.banner.base.scenario).toBe('fresh-install');
|
||||
expect(typeof parsed.banner.base.gbrain_version).toBe('string');
|
||||
// Same version on both sides (same process) → no version warning.
|
||||
expect(parsed.banner.warnings).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('identity normalization', () => {
|
||||
test('case, whitespace, and redaction differences collapse to the same identity', () => {
|
||||
startMarker('norm-a', { agent: 'scripted' });
|
||||
friction('norm-a', 'install', `Engine Init FAILED at ${process.cwd()}/x.ts`, 'error');
|
||||
startMarker('norm-b', { agent: 'scripted' });
|
||||
friction('norm-b', 'install', 'engine init failed at <CWD>/x.ts', 'error');
|
||||
|
||||
const diff = computeFrictionDiff('norm-a', 'norm-b');
|
||||
expect(diff.unique_to_base).toEqual([]);
|
||||
expect(diff.unique_to_compare).toEqual([]);
|
||||
expect(diff.changed).toEqual([]);
|
||||
});
|
||||
|
||||
test('messages sharing the first 80 normalized chars are the same identity', () => {
|
||||
const prefix = 'p'.repeat(80);
|
||||
startMarker('pre-a', { agent: 'scripted' });
|
||||
friction('pre-a', 'query', prefix + ' tail one', 'error');
|
||||
startMarker('pre-b', { agent: 'scripted' });
|
||||
friction('pre-b', 'query', prefix + ' completely different tail', 'error');
|
||||
|
||||
const diff = computeFrictionDiff('pre-a', 'pre-b');
|
||||
expect(diff.unique_to_base).toEqual([]);
|
||||
expect(diff.unique_to_compare).toEqual([]);
|
||||
expect(diff.changed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CLI dispatch', () => {
|
||||
test('diff with base and compare run-ids returns 0', () => {
|
||||
startMarker('cli-a', { agent: 'scripted' });
|
||||
startMarker('cli-b', { agent: 'scripted' });
|
||||
expect(runFriction(['diff', '--base', 'cli-a', '--compare', 'cli-b'])).toBe(0);
|
||||
});
|
||||
|
||||
test('missing compare flag is a usage error, exit 2', () => {
|
||||
const code = runFriction(['diff', '--base', 'cli-a']);
|
||||
expect(code).toBe(2);
|
||||
expect(stderrLines.join('')).toContain('usage');
|
||||
});
|
||||
|
||||
test('missing both flags is a usage error, exit 2', () => {
|
||||
expect(runFriction(['diff'])).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('identity hardening (adversarial-gate pins)', () => {
|
||||
test('a delight→friction flip surfaces as unique-to-each, never equality', () => {
|
||||
startMarker('run-base', { agent: 'scripted' });
|
||||
logFriction({ runId: 'run-base', phase: 'query', message: 'results ranked well', kind: 'delight', source: 'claw' });
|
||||
startMarker('run-cmp', { agent: 'scripted' });
|
||||
logFriction({ runId: 'run-cmp', phase: 'query', message: 'results ranked well', kind: 'friction', severity: 'confused', source: 'claw' });
|
||||
|
||||
const { code, parsed } = diffJson('run-base', 'run-cmp');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.changed).toEqual([]);
|
||||
expect(parsed.unique_to_base.length).toBe(1);
|
||||
expect(parsed.unique_to_base[0].kind).toBe('delight');
|
||||
expect(parsed.unique_to_compare.length).toBe(1);
|
||||
expect(parsed.unique_to_compare[0].kind).toBe('friction');
|
||||
});
|
||||
|
||||
test('severity redistribution with equal totals and equal severity sets is a reported difference', () => {
|
||||
startMarker('run-base', { agent: 'scripted' });
|
||||
friction('run-base', 'import', 'import warned', 'error');
|
||||
friction('run-base', 'import', 'import warned', 'error');
|
||||
friction('run-base', 'import', 'import warned', 'nit');
|
||||
startMarker('run-cmp', { agent: 'scripted' });
|
||||
friction('run-cmp', 'import', 'import warned', 'error');
|
||||
friction('run-cmp', 'import', 'import warned', 'nit');
|
||||
friction('run-cmp', 'import', 'import warned', 'nit');
|
||||
|
||||
const { code, parsed } = diffJson('run-base', 'run-cmp');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.changed.length).toBe(1);
|
||||
expect(parsed.changed[0].severity_changed).toBe(true);
|
||||
expect(parsed.changed[0].count_changed).toBe(false);
|
||||
expect(parsed.changed[0].base_severity_counts).toEqual({ error: 2, nit: 1 });
|
||||
expect(parsed.changed[0].compare_severity_counts).toEqual({ error: 1, nit: 2 });
|
||||
});
|
||||
|
||||
test('volatile digits (durations, retry counts) do not split identities across runs', () => {
|
||||
startMarker('run-base', { agent: 'scripted' });
|
||||
friction('run-base', 'agent_invoke', 'agent exited with code 1 after 5123ms');
|
||||
startMarker('run-cmp', { agent: 'scripted' });
|
||||
friction('run-cmp', 'agent_invoke', 'agent exited with code 1 after 98764ms');
|
||||
|
||||
const { code, parsed } = diffJson('run-base', 'run-cmp');
|
||||
expect(code).toBe(0);
|
||||
expect(parsed.unique_to_base).toEqual([]);
|
||||
expect(parsed.unique_to_compare).toEqual([]);
|
||||
expect(parsed.changed).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -25,7 +25,9 @@
|
||||
*
|
||||
* The drop-list is the security contract: CONDUCTOR_* / CLAUDE_* / GSTACK_* /
|
||||
* MCP_* / GBRAIN_* never reach a child except via the explicit overrides the
|
||||
* caller passes (which spread LAST and always win).
|
||||
* caller passes (which spread LAST and always win). HERMES_HOME is handled the
|
||||
* same way — not in any allowlist, so it only reaches a child via an explicit
|
||||
* override (the hermes door test always sets it to a temp home).
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -162,6 +164,24 @@ export function resolveClaudeBinary(): string | null {
|
||||
]);
|
||||
}
|
||||
|
||||
/** Locate the real `hermes` binary (NousResearch hermes-agent). Bun.which
|
||||
* first, then the installer's known landing spots. */
|
||||
export function resolveHermesBinary(): string | null {
|
||||
const which = whichBin('hermes');
|
||||
if (which) return which;
|
||||
const home = process.env.HOME ?? os.homedir();
|
||||
const candidates = [
|
||||
'/opt/homebrew/bin/hermes',
|
||||
'/usr/local/bin/hermes',
|
||||
`${home}/.local/bin/hermes`, // where the official installer symlinks (observed v0.20.0)
|
||||
`${home}/.hermes/bin/hermes`,
|
||||
];
|
||||
for (const dir of (process.env.PATH ?? '').split(path.delimiter)) {
|
||||
if (dir) candidates.push(path.join(dir, 'hermes'));
|
||||
}
|
||||
return firstExecutable(candidates);
|
||||
}
|
||||
|
||||
/** Locate the real `codex` binary. Bun.which first, then known install dirs
|
||||
* (adds ~/.nvm + common node bin dirs where the npm global lands). */
|
||||
export function resolveCodexBinary(): string | null {
|
||||
@@ -215,6 +235,56 @@ export function hasCodexAuth(): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/** Every provider key hermes recognizes — scrubbed from child env so the
|
||||
* seeded .env is the SINGLE auth source. Observed (v0.20.0): with model
|
||||
* pinned to anthropic/* but MULTIPLE provider keys visible, hermes's
|
||||
* provider-auto mis-routes the request and the turn returns
|
||||
* "HTTP 401: Missing Authentication header" as final text (exit 0). */
|
||||
const HERMES_ALL_PROVIDER_KEYS = [
|
||||
'ANTHROPIC_API_KEY', 'ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN',
|
||||
'OPENAI_API_KEY', 'OPENROUTER_API_KEY',
|
||||
] as const;
|
||||
|
||||
/** Parse KEY=VALUE lines from a dotenv-style file. Ignores comments, blanks,
|
||||
* and export prefixes; strips single/double quotes. Never throws. */
|
||||
export function parseDotenvFile(file: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
try {
|
||||
for (const rawLine of fs.readFileSync(file, 'utf-8').split('\n')) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) continue;
|
||||
const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
||||
if (!m) continue;
|
||||
let v = m[2].trim();
|
||||
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
|
||||
v = v.slice(1, -1);
|
||||
}
|
||||
out[m[1]] = v;
|
||||
}
|
||||
} catch {
|
||||
/* unreadable → empty */
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hermes is usable BY THE DOOR SUITE if an ANTHROPIC key with a NON-EMPTY
|
||||
* value is available — either exported (GSTACK_ promotion applies) or present
|
||||
* in the operator's real ~/.hermes/.env.
|
||||
*
|
||||
* Anthropic-only on purpose: the door pins model.default to anthropic/*, and
|
||||
* seeding any second provider key makes hermes's provider-auto mis-route the
|
||||
* pinned model ("HTTP 401: Missing Authentication header", observed). Bare
|
||||
* file existence is deliberately NOT auth: a blank CI secret writes an empty
|
||||
* .env, and that must produce a SKIP, not a paid failing test.
|
||||
*/
|
||||
export function hasHermesAuth(): boolean {
|
||||
const env = promotedEnv(process.env);
|
||||
if (env.ANTHROPIC_API_KEY?.trim()) return true;
|
||||
const parsed = parseDotenvFile(path.join(os.homedir(), '.hermes', '.env'));
|
||||
return Boolean(parsed.ANTHROPIC_API_KEY?.trim());
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// 4. Stream parsers (pure — exercised by the unit test with fixtures)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
@@ -471,6 +541,135 @@ export async function codexExecTurn(opts: CodexTurnOpts): Promise<CodexTurnResul
|
||||
};
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// 5a-bis. Hermes home seeding + one-shot turn (mirror of the codex trio)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SeedHermesHomeOpts {
|
||||
/** Test-only injection point: read provider keys from this dotenv file
|
||||
* instead of the operator's real ~/.hermes/.env (lets the unit test assert
|
||||
* the allowlist-only copy against a fixture without touching real homes). */
|
||||
sourceEnvPath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed a hermetic <home>/.hermes for a spawned hermes. Copies EXACTLY ONE
|
||||
* key — a non-empty ANTHROPIC_API_KEY, from the operator's real ~/.hermes/.env
|
||||
* when present, falling back to the (promoted) process env. One key on
|
||||
* purpose: the door pins an anthropic/* model, and a second provider key
|
||||
* flips hermes's provider-auto into a mis-routed request (observed 401).
|
||||
* Never the whole .env file (other creds / endpoints / behavior knobs stay
|
||||
* behind), NEVER config.yaml (the operator's private MCP servers). The model
|
||||
* pin is a separate step (`pinHermesModel`) because hermes owns config.yaml's
|
||||
* schema — hand-writing it risks drift; `hermes config set` round-trips
|
||||
* safely.
|
||||
*/
|
||||
export function seedHermesHome(home: string, opts?: SeedHermesHomeOpts): string {
|
||||
const hermesHome = path.join(home, '.hermes');
|
||||
fs.mkdirSync(hermesHome, { recursive: true });
|
||||
|
||||
const fromFile = parseDotenvFile(opts?.sourceEnvPath ?? path.join(os.homedir(), '.hermes', '.env'));
|
||||
const env = promotedEnv(process.env);
|
||||
const key = fromFile.ANTHROPIC_API_KEY?.trim() || env.ANTHROPIC_API_KEY?.trim();
|
||||
if (key) {
|
||||
fs.writeFileSync(path.join(hermesHome, '.env'), `ANTHROPIC_API_KEY=${key}\n`, { mode: 0o600 });
|
||||
}
|
||||
return hermesHome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hermetic env for spawning hermes itself: standard scrub + HOME/HERMES_HOME
|
||||
* overrides, then ALL provider keys deleted so the seeded .env is the single
|
||||
* auth source (provider-auto determinism — see HERMES_ALL_PROVIDER_KEYS).
|
||||
*/
|
||||
export function hermesChildEnv(home: string): NodeJS.ProcessEnv {
|
||||
const env = hermeticChildEnv({ HOME: home, HERMES_HOME: path.join(home, '.hermes') });
|
||||
for (const k of HERMES_ALL_PROVIDER_KEYS) delete env[k];
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-interactive model/provider pin for a hermetic hermes home. A virgin
|
||||
* install refuses `-z` with "No inference provider configured" (exit 1,
|
||||
* observed), and `hermes model` is interactive-only — `config set` is the
|
||||
* scriptable path (observed working against v0.20.0).
|
||||
*/
|
||||
export function pinHermesModel(hermesBin: string, home: string, model = 'anthropic/claude-haiku-4.5'): { code: number | null; stderr: string } {
|
||||
const res = spawnSync(hermesBin, ['config', 'set', 'model.default', model], {
|
||||
env: hermesChildEnv(home),
|
||||
encoding: 'utf8',
|
||||
timeout: 60_000,
|
||||
});
|
||||
return { code: res.status, stderr: res.stderr ?? '' };
|
||||
}
|
||||
|
||||
export interface HermesTurnOpts {
|
||||
prompt: string;
|
||||
cwd: string;
|
||||
home: string;
|
||||
timeoutMs?: number;
|
||||
/** When set, the turn passes hermes's usage-report flag targeting this path. */
|
||||
usageFile?: string;
|
||||
}
|
||||
|
||||
export interface HermesTurnResult {
|
||||
/** hermes's one-shot mode prints ONLY the final response text on stdout. */
|
||||
finalText: string;
|
||||
exitCode: number | null;
|
||||
timedOut: boolean;
|
||||
stderrText: string;
|
||||
/** Parsed usage-report JSON when usageFile was requested and parseable. */
|
||||
usage?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive one `hermes -z` turn against a hermetic HOME + HERMES_HOME. The
|
||||
* RESOLVED binary path is used (never the bare literal), so resolution and
|
||||
* execution can't disagree. stdout is plain final text — NOT NDJSON; there is
|
||||
* no per-event tool-call stream to parse (door tests use a negative-control
|
||||
* prompt instead).
|
||||
*/
|
||||
export async function hermesOneShotTurn(opts: HermesTurnOpts): Promise<HermesTurnResult> {
|
||||
const bin = resolveHermesBinary();
|
||||
if (!bin) throw new Error('hermesOneShotTurn: hermes binary not found');
|
||||
const timeoutMs = opts.timeoutMs ?? 240_000;
|
||||
|
||||
const argv = [bin, '-z', opts.prompt, ...(opts.usageFile ? ['--usage-file', opts.usageFile] : [])];
|
||||
const proc = Bun.spawn(argv, {
|
||||
cwd: opts.cwd,
|
||||
env: hermesChildEnv(opts.home),
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
stdin: 'ignore',
|
||||
});
|
||||
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
try { proc.kill(); } catch { /* already dead */ }
|
||||
}, timeoutMs);
|
||||
|
||||
const [stdout, stderrText] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text().catch(() => ''),
|
||||
]);
|
||||
const exitCode = await proc.exited;
|
||||
clearTimeout(timer);
|
||||
|
||||
let usage: unknown;
|
||||
if (opts.usageFile) {
|
||||
try { usage = JSON.parse(fs.readFileSync(opts.usageFile, 'utf-8')); } catch { /* best-effort */ }
|
||||
}
|
||||
|
||||
return {
|
||||
finalText: stdout.trim(),
|
||||
exitCode: timedOut ? 124 : exitCode,
|
||||
timedOut,
|
||||
stderrText,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// 5b. Fast gbrain MCP server command (compiled binary, cached; bun-run fallback)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -16,13 +16,21 @@
|
||||
* leaks into sibling tests.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
parseClaudeStream,
|
||||
parseCodexJsonl,
|
||||
hermeticChildEnv,
|
||||
hermesChildEnv,
|
||||
promotedEnv,
|
||||
resolveClaudeBinary,
|
||||
resolveCodexBinary,
|
||||
resolveHermesBinary,
|
||||
hasHermesAuth,
|
||||
parseDotenvFile,
|
||||
seedHermesHome,
|
||||
} from './agent-harness.ts';
|
||||
import { withEnv } from './with-env.ts';
|
||||
|
||||
@@ -173,4 +181,156 @@ describe('binary resolution SMOKE', () => {
|
||||
expect(bin === null || typeof bin === 'string').toBe(true);
|
||||
if (bin) console.log(`[smoke] codex resolved at: ${bin}`);
|
||||
});
|
||||
|
||||
test('resolveHermesBinary returns a string or null', () => {
|
||||
const bin = resolveHermesBinary();
|
||||
expect(bin === null || typeof bin === 'string').toBe(true);
|
||||
if (bin) console.log(`[smoke] hermes resolved at: ${bin}`);
|
||||
});
|
||||
|
||||
test('hasHermesAuth returns a boolean', () => {
|
||||
expect(typeof hasHermesAuth()).toBe('boolean');
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasHermesAuth truth table (env leg — pins the anthropic-only, non-empty-value gate)', () => {
|
||||
// The .env-file leg reads the operator's real ~/.hermes/.env, so only the
|
||||
// env-var leg is exercised hermetically here; the file PARSING contract is
|
||||
// pinned by the parseDotenvFile + seedHermesHome describes below.
|
||||
const CLEAR = {
|
||||
ANTHROPIC_API_KEY: undefined,
|
||||
GSTACK_ANTHROPIC_API_KEY: undefined,
|
||||
OPENAI_API_KEY: undefined,
|
||||
GSTACK_OPENAI_API_KEY: undefined,
|
||||
OPENROUTER_API_KEY: undefined,
|
||||
} as const;
|
||||
|
||||
/** True only when the operator's real ~/.hermes/.env carries an anthropic key. */
|
||||
const fileLegHasAnthropicKey = () =>
|
||||
Boolean(parseDotenvFile(join(process.env.HOME ?? '', '.hermes', '.env')).ANTHROPIC_API_KEY?.trim());
|
||||
|
||||
test('non-empty anthropic env key → true', async () => {
|
||||
await withEnv({ ...CLEAR, ANTHROPIC_API_KEY: 'sk-test-nonempty' }, () => {
|
||||
expect(hasHermesAuth()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('GSTACK_-prefixed anthropic key promotes → true', async () => {
|
||||
await withEnv({ ...CLEAR, GSTACK_ANTHROPIC_API_KEY: 'sk-test-promoted' }, () => {
|
||||
expect(hasHermesAuth()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('BLANK env value → does NOT count as auth (a blank CI secret must skip, not fail paid)', async () => {
|
||||
await withEnv({ ...CLEAR, ANTHROPIC_API_KEY: ' ' }, () => {
|
||||
expect(hasHermesAuth()).toBe(fileLegHasAnthropicKey());
|
||||
});
|
||||
});
|
||||
|
||||
test('a NON-anthropic provider key alone → false (door is anthropic-pinned; a second provider mis-routes provider-auto)', async () => {
|
||||
await withEnv({ ...CLEAR, OPENAI_API_KEY: 'sk-openai-only' }, () => {
|
||||
expect(hasHermesAuth()).toBe(fileLegHasAnthropicKey());
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('hermesChildEnv — the single-auth-source enforcement point', () => {
|
||||
test('scrubs EVERY provider key (incl. allowlisted anthropic vars) and pins HOME/HERMES_HOME', async () => {
|
||||
// A regression here (dropping the delete loop) re-leaks the operator's
|
||||
// real key into every hermes child and reintroduces the provider-auto
|
||||
// 401 mis-route — detectable only in the triple-gated paid lane, so the
|
||||
// contract is pinned hermetically here.
|
||||
await withEnv({
|
||||
ANTHROPIC_API_KEY: 'sk-operator',
|
||||
ANTHROPIC_BASE_URL: 'https://operator.example',
|
||||
ANTHROPIC_AUTH_TOKEN: 'tok-operator',
|
||||
OPENAI_API_KEY: 'sk-openai',
|
||||
OPENROUTER_API_KEY: 'sk-openrouter',
|
||||
GSTACK_ANTHROPIC_API_KEY: undefined,
|
||||
GSTACK_OPENAI_API_KEY: undefined,
|
||||
HERMES_HOME: '/operator/.hermes',
|
||||
}, () => {
|
||||
const env = hermesChildEnv('/tmp/hermes-child-test');
|
||||
for (const k of ['ANTHROPIC_API_KEY', 'ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN', 'OPENAI_API_KEY', 'OPENROUTER_API_KEY']) {
|
||||
expect(env[k]).toBeUndefined();
|
||||
}
|
||||
expect(env.HOME).toBe('/tmp/hermes-child-test');
|
||||
expect(env.HERMES_HOME).toBe('/tmp/hermes-child-test/.hermes');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDotenvFile', () => {
|
||||
test('parses KEY=VALUE, skips comments/blanks, strips quotes and export prefixes', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gb-dotenv-'));
|
||||
try {
|
||||
const f = join(dir, '.env');
|
||||
writeFileSync(f, [
|
||||
'# comment',
|
||||
'',
|
||||
'ANTHROPIC_API_KEY=sk-plain',
|
||||
'export OPENAI_API_KEY="sk-quoted"',
|
||||
"OPENROUTER_API_KEY='sk-single'",
|
||||
'BLANK_KEY=',
|
||||
'not a kv line',
|
||||
].join('\n'), 'utf-8');
|
||||
const parsed = parseDotenvFile(f);
|
||||
expect(parsed.ANTHROPIC_API_KEY).toBe('sk-plain');
|
||||
expect(parsed.OPENAI_API_KEY).toBe('sk-quoted');
|
||||
expect(parsed.OPENROUTER_API_KEY).toBe('sk-single');
|
||||
expect(parsed.BLANK_KEY).toBe('');
|
||||
expect(Object.keys(parsed)).not.toContain('not a kv line');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('unreadable file → empty object, never throws', () => {
|
||||
expect(parseDotenvFile('/nonexistent/path/.env')).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('seedHermesHome single-key copy (injectable source — never the operator home)', () => {
|
||||
test('copies EXACTLY the anthropic key; other providers and behavior knobs stay behind', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gb-seedhh-'));
|
||||
try {
|
||||
const src = join(dir, 'source.env');
|
||||
writeFileSync(src, [
|
||||
'ANTHROPIC_API_KEY=sk-copy-me',
|
||||
'OPENAI_API_KEY=sk-openai-stays-home', // second provider → dropped (mis-routes provider-auto)
|
||||
'OPENROUTER_API_KEY=sk-or-stays-home', // second provider → dropped
|
||||
'TELEGRAM_BOT_TOKEN=secret-stays-home', // unlisted → dropped
|
||||
'HERMES_BASE_URL=https://internal', // unlisted → dropped
|
||||
].join('\n'), 'utf-8');
|
||||
|
||||
const home = join(dir, 'home');
|
||||
const hermesHome = await withEnv({
|
||||
ANTHROPIC_API_KEY: undefined, GSTACK_ANTHROPIC_API_KEY: undefined,
|
||||
OPENAI_API_KEY: undefined, GSTACK_OPENAI_API_KEY: undefined, OPENROUTER_API_KEY: undefined,
|
||||
}, () => seedHermesHome(home, { sourceEnvPath: src }));
|
||||
|
||||
expect(hermesHome).toBe(join(home, '.hermes'));
|
||||
const written = readFileSync(join(hermesHome, '.env'), 'utf-8');
|
||||
expect(written).toBe('ANTHROPIC_API_KEY=sk-copy-me\n');
|
||||
// seedHermesHome never writes config.yaml (hermes owns that schema —
|
||||
// the model pin goes through the hermes CLI instead).
|
||||
expect(existsSync(join(hermesHome, 'config.yaml'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('no source file + no env keys → no .env written (gate stays closed)', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gb-seedhh2-'));
|
||||
try {
|
||||
const home = join(dir, 'home');
|
||||
await withEnv({
|
||||
ANTHROPIC_API_KEY: undefined, GSTACK_ANTHROPIC_API_KEY: undefined,
|
||||
OPENAI_API_KEY: undefined, GSTACK_OPENAI_API_KEY: undefined, OPENROUTER_API_KEY: undefined,
|
||||
}, () => seedHermesHome(home, { sourceEnvPath: join(dir, 'missing.env') }));
|
||||
expect(existsSync(join(home, '.hermes', '.env'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -99,6 +99,76 @@ describe('loadScenario', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadScenario — oracle validation', () => {
|
||||
const base = (oracle: string) =>
|
||||
`{"kind":"fresh-install","expected_phases":[],"oracle":${oracle}}`;
|
||||
|
||||
test('happy path: query defaults minResults to 1; min_results overrides; files_exist parsed', () => {
|
||||
scaffoldScenario('o-happy', base('{"query":"alice"}'));
|
||||
expect(loadScenario('o-happy').oracle).toEqual({ query: 'alice', minResults: 1 });
|
||||
|
||||
scaffoldScenario('o-min', base('{"query":"alice","min_results":0,"files_exist":["skills/query/SKILL.md"]}'));
|
||||
const cfg = loadScenario('o-min');
|
||||
expect(cfg.oracle?.minResults).toBe(0);
|
||||
expect(cfg.oracle?.filesExist).toEqual(['skills/query/SKILL.md']);
|
||||
});
|
||||
|
||||
test('oracle must be a JSON object', () => {
|
||||
scaffoldScenario('o-arr', base('["x"]'));
|
||||
expect(() => loadScenario('o-arr')).toThrow(/oracle must be a JSON object/);
|
||||
scaffoldScenario('o-str', base('"x"'));
|
||||
expect(() => loadScenario('o-str')).toThrow(/oracle must be a JSON object/);
|
||||
});
|
||||
|
||||
test('oracle.query must be a non-empty string and must not start with a dash', () => {
|
||||
scaffoldScenario('o-empty', base('{"query":" "}'));
|
||||
expect(() => loadScenario('o-empty')).toThrow(/oracle\.query must be a non-empty string/);
|
||||
scaffoldScenario('o-num', base('{"query":7}'));
|
||||
expect(() => loadScenario('o-num')).toThrow(/oracle\.query must be a non-empty string/);
|
||||
// The query lands in gbrain argv — a dash-leading value would parse as a flag.
|
||||
scaffoldScenario('o-dash', base('{"query":"--json"}'));
|
||||
expect(() => loadScenario('o-dash')).toThrow(/must not start with a dash/);
|
||||
});
|
||||
|
||||
test('oracle.min_results requires query and must be a finite number >= 0', () => {
|
||||
scaffoldScenario('o-orphan', base('{"min_results":1}'));
|
||||
expect(() => loadScenario('o-orphan')).toThrow(/min_results requires oracle\.query/);
|
||||
scaffoldScenario('o-neg', base('{"query":"alice","min_results":-1}'));
|
||||
expect(() => loadScenario('o-neg')).toThrow(/min_results must be a number >= 0/);
|
||||
scaffoldScenario('o-nan', base('{"query":"alice","min_results":"5"}'));
|
||||
expect(() => loadScenario('o-nan')).toThrow(/min_results must be a number >= 0/);
|
||||
});
|
||||
|
||||
test('oracle.files_exist must be workspace-relative strings', () => {
|
||||
scaffoldScenario('o-files-str', base('{"files_exist":"skills"}'));
|
||||
expect(() => loadScenario('o-files-str')).toThrow(/files_exist must be a string\[\]/);
|
||||
scaffoldScenario('o-files-abs', base('{"files_exist":["/etc/passwd"]}'));
|
||||
expect(() => loadScenario('o-files-abs')).toThrow(/workspace-relative/);
|
||||
scaffoldScenario('o-files-dots', base('{"files_exist":["../outside.md"]}'));
|
||||
expect(() => loadScenario('o-files-dots')).toThrow(/workspace-relative/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadScenario — containment (adversarial-gate pins)', () => {
|
||||
// Scenario packs load from arbitrary dirs via the scenarios-dir env
|
||||
// override, and their content flows to an external agent — traversal in the
|
||||
// name or the declared paths would expose arbitrary operator files.
|
||||
test('traversal-shaped scenario names are rejected', () => {
|
||||
expect(() => loadScenario('../outside')).toThrow(/not found/);
|
||||
expect(() => loadScenario('a/b')).toThrow(/not found/);
|
||||
expect(() => loadScenario('..')).toThrow(/not found/);
|
||||
});
|
||||
|
||||
test('brief, brain, and seed paths escaping the scenario dir are rejected', () => {
|
||||
scaffoldScenario('esc-brief', '{"kind":"fresh-install","expected_phases":[],"brief":"../outside.md"}');
|
||||
expect(() => loadScenario('esc-brief')).toThrow(/inside the scenario dir/);
|
||||
scaffoldScenario('esc-brain', '{"kind":"fresh-install","expected_phases":[],"brain":"../../brains"}');
|
||||
expect(() => loadScenario('esc-brain')).toThrow(/inside the scenario dir/);
|
||||
scaffoldScenario('esc-seed', '{"kind":"upgrade","expected_phases":[],"seed":"/etc"}');
|
||||
expect(() => loadScenario('esc-seed')).toThrow(/inside the scenario dir/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readBrief', () => {
|
||||
test('returns BRIEF.md content', () => {
|
||||
scaffoldScenario('reads-brief', '{"kind":"fresh-install","expected_phases":[]}', '# Hello world');
|
||||
|
||||
Reference in New Issue
Block a user