mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 01:42:23 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
566d3e2f75 | ||
|
|
bf56e8cb8e |
-12
@@ -135,18 +135,6 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### Docker network isolation (self-hosted Postgres)
|
||||
|
||||
OAuth and source scoping enforce isolation on the `serve --http` path only.
|
||||
Raw Postgres reachability bypasses both: a container that shares Docker's
|
||||
default `bridge` network with the brain's Postgres can open a direct DB
|
||||
session without any token and read every source. Put the brain's Postgres on
|
||||
a user-defined Docker network with nothing untrusted on it, publish its port
|
||||
loopback-only (if at all), and never put `DATABASE_URL` or a Postgres
|
||||
password in untrusted agent containers — those should reach the brain
|
||||
exclusively via OAuth against `serve --http`. Full operator checklist:
|
||||
[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
|
||||
@@ -258,43 +258,6 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -484,10 +484,6 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho
|
||||
|
||||
The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard.
|
||||
|
||||
### If agents run as containers on the same Docker host
|
||||
|
||||
OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
---
|
||||
|
||||
## Part 13: Cost and speed expectations
|
||||
|
||||
@@ -3905,43 +3905,6 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -529,6 +529,61 @@ export async function childTableOrphansCheck(engine: BrainEngine): Promise<Check
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw-source persistence guarantee (#1978, warn-only v1).
|
||||
*
|
||||
* Invariant: every synthesized/derived page (dream_generated:true frontmatter
|
||||
* or type:synthesis) must either carry a raw trace or declare an explicit
|
||||
* exemption. Accepted traces:
|
||||
* - frontmatter key `raw_trace` / `raw_source` / `source_uri`
|
||||
* - an attached `raw_data` row
|
||||
* - `synthesis_evidence` rows (think-op citations)
|
||||
* - explicit `raw_trace_exempt: true` (reason in `raw_trace_exempt_reason`)
|
||||
*
|
||||
* v1 is deliberately warn-only — no write path is blocked. Escalation to
|
||||
* fail-closed enforcement in the synthesis/import write paths is the v2
|
||||
* follow-up once real brains run clean.
|
||||
*
|
||||
* Pure helper (engine.executeRaw only) for parity with
|
||||
* childTableOrphansCheck so tests can target it directly.
|
||||
*/
|
||||
export async function rawProvenanceCheck(engine: BrainEngine): Promise<Check> {
|
||||
const where = `
|
||||
p.deleted_at IS NULL
|
||||
AND (COALESCE(p.frontmatter->>'dream_generated', '') = 'true' OR p.type = 'synthesis')
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ?| ARRAY['raw_trace', 'raw_source', 'source_uri', 'raw_trace_exempt'])
|
||||
AND NOT EXISTS (SELECT 1 FROM raw_data rd WHERE rd.page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM synthesis_evidence se WHERE se.synthesis_page_id = p.id)`;
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ n: string | number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages p WHERE ${where}`,
|
||||
);
|
||||
const n = Number(rows[0]?.n ?? 0);
|
||||
if (n === 0) {
|
||||
return {
|
||||
name: 'raw_provenance',
|
||||
status: 'ok',
|
||||
message: 'All synthesized pages carry a raw trace or explicit exemption',
|
||||
};
|
||||
}
|
||||
const sample = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT p.slug FROM pages p WHERE ${where} ORDER BY p.slug LIMIT 5`,
|
||||
);
|
||||
const slugs = sample.map(r => r.slug).join(', ');
|
||||
return {
|
||||
name: 'raw_provenance',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${n} synthesized page(s) lack a raw trace (no raw_trace/raw_source/source_uri frontmatter, ` +
|
||||
`raw_data row, or synthesis evidence) and carry no raw_trace_exempt marker. e.g. ${slugs}. ` +
|
||||
`Fix: stamp raw_source (path/URI of the source material) or raw_trace_exempt: true + ` +
|
||||
`raw_trace_exempt_reason in frontmatter. Warn-only (#1978).`,
|
||||
};
|
||||
} catch {
|
||||
return { name: 'raw_provenance', status: 'warn', message: 'Could not check raw provenance (older schema?)' };
|
||||
}
|
||||
}
|
||||
|
||||
export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorReport> {
|
||||
const checks: Check[] = [];
|
||||
|
||||
@@ -6290,6 +6345,12 @@ export async function buildChecks(
|
||||
progress.heartbeat('child_table_orphans');
|
||||
checks.push(await childTableOrphansCheck(engine));
|
||||
|
||||
// 10d. Raw-source persistence guarantee (#1978, warn-only v1).
|
||||
// Every synthesized/derived page must carry a raw trace or an explicit
|
||||
// exemption. Warn-only in v1 — surfaces violations, blocks nothing.
|
||||
progress.heartbeat('raw_provenance');
|
||||
checks.push(await rawProvenanceCheck(engine));
|
||||
|
||||
// v0.33: whoknows_health — fixture presence + row count. The eval
|
||||
// gate itself runs via `gbrain eval whoknows`; this check is the
|
||||
// "did you do the assignment?" signal.
|
||||
|
||||
+20
-3
@@ -249,6 +249,13 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
|
||||
|
||||
// --- Tier 1+2: explicit flags ---------------------------------------------
|
||||
|
||||
// #2301: an explicit embedding flag on THIS invocation overrides the
|
||||
// persisted deferred-setup sentinel above. Without this, a stale
|
||||
// `embedding_disabled: true` in config.json made every re-init defer
|
||||
// embedding — including `gbrain init --embedding-model ...`, the exact
|
||||
// recovery path the deferred-setup message tells users to take.
|
||||
if (verbose || shorthand) delete out.noEmbedding;
|
||||
|
||||
if (verbose) {
|
||||
out.embedding_model = verbose;
|
||||
} else if (shorthand) {
|
||||
@@ -435,7 +442,7 @@ function printNoEmbeddingProviderHint(typos: Array<{ userSet: string; suggested:
|
||||
console.error(' gbrain init --pglite --embedding-model openai:text-embedding-3-large');
|
||||
console.error('');
|
||||
console.error('Or defer setup: gbrain init --pglite --no-embedding');
|
||||
console.error(' (you can configure later with `gbrain config set embedding_model <id>`)');
|
||||
console.error(' (you can configure later with `gbrain init --force --embedding-model <provider>:<model>`)');
|
||||
// D13: surface near-miss env vars (e.g. OPENAPI_API_KEY → OPENAI_API_KEY).
|
||||
if (typos.length > 0) {
|
||||
console.error('');
|
||||
@@ -833,7 +840,7 @@ async function initPGLite(opts: {
|
||||
let resolvedModel: string | undefined;
|
||||
if (opts.aiOpts?.noEmbedding) {
|
||||
// D9 deferred-setup mode: skip preflight, no model/dim resolved.
|
||||
console.log(` --no-embedding: deferred setup — configure with \`gbrain config set embedding_model <id>\` before import`);
|
||||
console.log(` --no-embedding: deferred setup — run \`gbrain init --force --embedding-model <provider>:<model>\` before import`);
|
||||
} else if (opts.aiOpts?.embedding_model) {
|
||||
const { resolveSchemaEmbeddingDim } = await import('../core/embedding-dim-check.ts');
|
||||
const pre = resolveSchemaEmbeddingDim({
|
||||
@@ -972,6 +979,12 @@ async function initPGLite(opts: {
|
||||
// unless explicitly overridden by --schema-pack on re-init.
|
||||
...(opts.schemaPack ? { schema_pack: opts.schemaPack } : {}),
|
||||
};
|
||||
// #2301: a resolved embedding model supersedes any stale deferred-setup
|
||||
// sentinel carried over via ...existingFile — otherwise the sentinel
|
||||
// re-defers embedding on every future init/embed forever.
|
||||
if (!opts.aiOpts?.noEmbedding && resolvedModel && resolvedDim) {
|
||||
delete config.embedding_disabled;
|
||||
}
|
||||
// PR1: new installs publish their skill catalog over MCP by default
|
||||
// (existing config wins on re-init, so a prior opt-out is preserved).
|
||||
config.mcp = { publish_skills: true, ...(config.mcp ?? {}) };
|
||||
@@ -1056,7 +1069,7 @@ async function initPostgres(opts: {
|
||||
let resolvedDim: number | undefined;
|
||||
let resolvedModel: string | undefined;
|
||||
if (opts.aiOpts?.noEmbedding) {
|
||||
console.log(` --no-embedding: deferred setup — configure with \`gbrain config set embedding_model <id>\` before import`);
|
||||
console.log(` --no-embedding: deferred setup — run \`gbrain init --force --embedding-model <provider>:<model>\` before import`);
|
||||
} else if (opts.aiOpts?.embedding_model) {
|
||||
const { resolveSchemaEmbeddingDim } = await import('../core/embedding-dim-check.ts');
|
||||
const pre = resolveSchemaEmbeddingDim({
|
||||
@@ -1220,6 +1233,10 @@ async function initPostgres(opts: {
|
||||
// v0.42 (T17): same schema_pack default as PGLite path.
|
||||
...(opts.schemaPack ? { schema_pack: opts.schemaPack } : {}),
|
||||
};
|
||||
// #2301: same stale-sentinel drop as the PGLite path above.
|
||||
if (!opts.aiOpts?.noEmbedding && resolvedModel && resolvedDim) {
|
||||
delete config.embedding_disabled;
|
||||
}
|
||||
// PR1: new installs publish their skill catalog over MCP by default
|
||||
// (existing config wins on re-init, so a prior opt-out is preserved).
|
||||
config.mcp = { publish_skills: true, ...(config.mcp ?? {}) };
|
||||
|
||||
@@ -486,44 +486,9 @@ const DEFAULT_PARALLELISM = 4;
|
||||
* src/core/errors.ts (the v0.19.0 envelope every new agent-facing
|
||||
* surface uses) rather than introducing a new BrainstormError class.
|
||||
*/
|
||||
/** File-config slice the orchestrator reads (see loadConfig in core/config.ts). */
|
||||
export interface BrainstormRunConfig {
|
||||
embedding_model?: string;
|
||||
chat_model?: string;
|
||||
emotional_weight?: { user_holder?: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Model used for the cost preview + hard cost ceiling. Mirrors what the
|
||||
* gateway will actually run: explicit --model override, else the configured
|
||||
* chat_model (gateway default), else the hardcoded gateway fallback. Before
|
||||
* this resolved through config, a non-Sonnet chat_model got its preview
|
||||
* priced against the wrong model. (Takeover of PR #1855 by @starm2010.)
|
||||
*/
|
||||
export function resolveBrainstormChatModel(
|
||||
config: { chat_model?: string },
|
||||
modelOverride?: string,
|
||||
): string {
|
||||
return modelOverride ?? config.chat_model ?? 'anthropic:claude-sonnet-4-6';
|
||||
}
|
||||
|
||||
/**
|
||||
* Judge-phase model precedence: --judge-model flag, else the
|
||||
* `models.brainstorm.judge` config key, else undefined (falls back to
|
||||
* `modelOverride` then the gateway default at the runJudge callsite).
|
||||
*/
|
||||
export async function resolveBrainstormJudgeModel(
|
||||
engine: BrainEngine,
|
||||
judgeModelFlag?: string,
|
||||
): Promise<string | undefined> {
|
||||
if (judgeModelFlag) return judgeModelFlag;
|
||||
const configured = await engine.getConfig('models.brainstorm.judge');
|
||||
return configured ?? undefined;
|
||||
}
|
||||
|
||||
export async function runBrainstorm(
|
||||
engine: BrainEngine,
|
||||
config: BrainstormRunConfig,
|
||||
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
|
||||
opts: BrainstormOptions
|
||||
): Promise<BrainstormResult> {
|
||||
// v0.39.3.0 (Phase 5, CV11+T4): outer try/catch around the orchestrator
|
||||
@@ -545,7 +510,7 @@ export async function runBrainstorm(
|
||||
|
||||
async function runBrainstormImpl(
|
||||
engine: BrainEngine,
|
||||
config: BrainstormRunConfig,
|
||||
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
|
||||
opts: BrainstormOptions,
|
||||
): Promise<BrainstormResult> {
|
||||
// v0.39.0.0 T10: install a gateway-layer BudgetTracker scope around the
|
||||
@@ -565,7 +530,7 @@ async function runBrainstormImpl(
|
||||
|
||||
async function _runBrainstormInner(
|
||||
engine: BrainEngine,
|
||||
config: BrainstormRunConfig,
|
||||
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
|
||||
opts: BrainstormOptions,
|
||||
): Promise<BrainstormResult> {
|
||||
const profile = opts.profile ?? BRAINSTORM_PROFILE;
|
||||
@@ -574,7 +539,7 @@ async function _runBrainstormInner(
|
||||
const embedFn = opts.embedQueryFn ?? embedQuery;
|
||||
|
||||
// ---- Phase 0: cost preview + TTY grace ----
|
||||
const modelStr = resolveBrainstormChatModel(config, opts.modelOverride);
|
||||
const modelStr = opts.modelOverride ?? 'anthropic:claude-sonnet-4-6';
|
||||
const { aborted, estimate } = await previewCostAndWait({
|
||||
profile,
|
||||
model: modelStr,
|
||||
@@ -883,7 +848,7 @@ async function _runBrainstormInner(
|
||||
far_slug: i.far_slug,
|
||||
}));
|
||||
const judgeResult = await runJudge(profile.judge_config, judgeInput, {
|
||||
modelOverride: (await resolveBrainstormJudgeModel(engine, opts.judgeModel)) ?? opts.modelOverride,
|
||||
modelOverride: opts.judgeModel ?? opts.modelOverride,
|
||||
chatFn: opts.chatFn,
|
||||
activeBiasTags: activeBiasTags ?? undefined,
|
||||
abortSignal: opts.abortSignal,
|
||||
|
||||
@@ -962,7 +962,6 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
|
||||
'models.subagent',
|
||||
'models.expansion',
|
||||
'models.chat',
|
||||
'models.brainstorm.judge',
|
||||
'models.eval.longmemeval',
|
||||
'facts.extraction_model',
|
||||
// #2113: output-token cap for the per-turn facts extractor (default 4000).
|
||||
|
||||
@@ -39,11 +39,11 @@
|
||||
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { chat as gatewayChat, getChatModel, probeChatModel } from '../ai/gateway.ts';
|
||||
import { normalizeModelId } from '../model-id.ts';
|
||||
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
|
||||
import { writeReceipt } from '../extract/receipt-writer.ts';
|
||||
import { upsertExtractRollup } from '../extract/rollup-writer.ts';
|
||||
import { GBrainError } from '../types.ts';
|
||||
import type { Page, PageFilters } from '../types.ts';
|
||||
import type { OperationContext } from '../operations.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { PhaseStatus, CyclePhase } from '../cycle.ts';
|
||||
@@ -160,48 +160,6 @@ export interface ProposeTakesResult {
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/** Narrow projection of `pages` — the only columns this phase reads. */
|
||||
interface ProposeTakesPageRow {
|
||||
slug: string;
|
||||
source_id: string;
|
||||
compiled_truth: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load proposal candidates with a narrow projection instead of
|
||||
* `engine.listPages` (`SELECT p.*`). The phase only reads slug, source_id
|
||||
* and compiled_truth — skipping timeline/frontmatter/title keeps large
|
||||
* toasted columns out of the hot path. Scope precedence mirrors
|
||||
* `sourceScopeOpts`: federated array (`sourceIds`) beats scalar
|
||||
* (`sourceId`); ordering matches `PAGE_SORT_SQL.updated_desc` with an id
|
||||
* tiebreak for determinism. (Takeover of PR #1979's projection by
|
||||
* @shawnduggan.)
|
||||
*/
|
||||
async function listCandidatePages(
|
||||
engine: BrainEngine,
|
||||
scope: ScopedReadOpts,
|
||||
limit: number,
|
||||
): Promise<ProposeTakesPageRow[]> {
|
||||
const where = ['deleted_at IS NULL'];
|
||||
const params: unknown[] = [];
|
||||
if (scope.sourceIds && scope.sourceIds.length > 0) {
|
||||
params.push(scope.sourceIds);
|
||||
where.push(`source_id = ANY($${params.length}::text[])`);
|
||||
} else if (scope.sourceId) {
|
||||
params.push(scope.sourceId);
|
||||
where.push(`source_id = $${params.length}`);
|
||||
}
|
||||
params.push(limit);
|
||||
return engine.executeRaw<ProposeTakesPageRow>(
|
||||
`SELECT slug, source_id, compiled_truth
|
||||
FROM pages
|
||||
WHERE ${where.join(' AND ')}
|
||||
ORDER BY updated_at DESC, id DESC
|
||||
LIMIT $${params.length}`,
|
||||
params,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the content_hash key for the idempotency cache. SHA-256 of the
|
||||
* page body suffices — page slug + prompt_version are separate columns in
|
||||
@@ -372,34 +330,6 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
const phaseStartMs = Date.now();
|
||||
const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`;
|
||||
|
||||
const modelId = opts.model ?? getChatModel();
|
||||
|
||||
// With the default (gateway) extractor, skip cheaply when the resolved
|
||||
// model's provider can't run — same probe semantics as patterns.ts /
|
||||
// think/index.ts: unknown provider/model or Anthropic-without-key skips;
|
||||
// other providers' auth surfaces lazily at chat() time. An injected
|
||||
// extractor bypasses the gateway, so it is never gated. (Takeover of
|
||||
// PR #1979's intent by @shawnduggan.)
|
||||
if (!opts.extractor) {
|
||||
const probe = probeChatModel(normalizeModelId(modelId));
|
||||
if (!probe.ok) {
|
||||
return {
|
||||
summary: `propose_takes skipped: ${probe.detail}`,
|
||||
details: {
|
||||
reason: 'no_provider',
|
||||
model: modelId,
|
||||
pages_scanned: 0,
|
||||
cache_hits: 0,
|
||||
cache_misses: 0,
|
||||
proposals_inserted: 0,
|
||||
budget_exhausted: false,
|
||||
warnings: [],
|
||||
},
|
||||
status: 'skipped',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const result: ProposeTakesResult = {
|
||||
pages_scanned: 0,
|
||||
cache_hits: 0,
|
||||
@@ -410,12 +340,19 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
};
|
||||
|
||||
// Load pages eligible for proposal. Source-scoped per BaseCyclePhase.
|
||||
const pages = await listCandidatePages(engine, scope, pageLimit);
|
||||
const pageFilters: PageFilters = {
|
||||
...scope,
|
||||
limit: pageLimit,
|
||||
sort: 'updated_desc',
|
||||
};
|
||||
const pages: Page[] = await engine.listPages(pageFilters);
|
||||
|
||||
if (opts.reporter) {
|
||||
opts.reporter.start('propose_takes.pages' as never, pages.length);
|
||||
}
|
||||
|
||||
const modelId = opts.model ?? getChatModel();
|
||||
|
||||
for (const page of pages) {
|
||||
// Phase deadline check. Break (not throw) so the phase returns a
|
||||
// partial result with deadline_hit:true; work already banked stays.
|
||||
@@ -572,5 +509,4 @@ export const __testing = {
|
||||
contentHash,
|
||||
hasCompleteFence,
|
||||
extractExistingTakesForDedup,
|
||||
listCandidatePages,
|
||||
};
|
||||
|
||||
@@ -554,6 +554,8 @@ export async function runPhaseSynthesize(
|
||||
const childIds: number[] = [];
|
||||
/** Map child job_id → chunk metadata for D6 orchestrator-side slug rewrite. */
|
||||
const chunkInfo = new Map<number, { idx: number; hash6: string }>();
|
||||
/** #1978: map child job_id → source transcript path so written pages get a raw_source stamp. */
|
||||
const jobRawSource = new Map<number, string>();
|
||||
/** Skip reasons for the cycle report (D5 cap hits, D8 legacy-key skips). */
|
||||
const skipReports: Array<{ filePath: string; reason: string }> = [];
|
||||
|
||||
@@ -638,6 +640,7 @@ export async function runPhaseSynthesize(
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
childIds.push(child.id);
|
||||
jobRawSource.set(child.id, t.filePath);
|
||||
if (isChunked) {
|
||||
chunkInfo.set(child.id, { idx: i, hash6 });
|
||||
}
|
||||
@@ -682,7 +685,7 @@ export async function runPhaseSynthesize(
|
||||
// (source, slug) row. #1586: refs are stamped with the cycle's resolved
|
||||
// source (children write there via SubagentHandlerData.source_id).
|
||||
const cycleSourceId = opts.sourceId ?? 'default';
|
||||
const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId);
|
||||
const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId, jobRawSource);
|
||||
|
||||
const summaryDate = opts.date ?? today();
|
||||
|
||||
@@ -1234,7 +1237,8 @@ async function collectChildPutPageSlugs(
|
||||
childIds: number[],
|
||||
chunkInfo: Map<number, { idx: number; hash6: string }>,
|
||||
sourceId = 'default',
|
||||
): Promise<Array<{ slug: string; source_id: string }>> {
|
||||
jobRawSource?: Map<number, string>,
|
||||
): Promise<Array<{ slug: string; source_id: string; raw_source?: string }>> {
|
||||
if (childIds.length === 0) return [];
|
||||
// Raw fetch — NO SELECT DISTINCT. Preserves per-child slug duplicates so
|
||||
// the orchestrator sees what each child wrote. COALESCE handles both
|
||||
@@ -1256,13 +1260,21 @@ async function collectChildPutPageSlugs(
|
||||
AND status = 'complete'`,
|
||||
[childIds],
|
||||
);
|
||||
const rewritten = new Set<string>();
|
||||
// #1978: slug → source transcript path (first writer wins) so the
|
||||
// provenance stamp can record WHERE the synthesized content came from.
|
||||
const rewritten = new Map<string, string | undefined>();
|
||||
for (const r of rows) {
|
||||
if (typeof r.slug !== 'string' || r.slug.length === 0) continue;
|
||||
const ci = chunkInfo.get(r.job_id);
|
||||
rewritten.add(ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug);
|
||||
const slug = ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug;
|
||||
if (!rewritten.has(slug) || rewritten.get(slug) === undefined) {
|
||||
rewritten.set(slug, jobRawSource?.get(r.job_id));
|
||||
}
|
||||
}
|
||||
return Array.from(rewritten).sort().map(slug => ({ slug, source_id: sourceId }));
|
||||
return Array.from(rewritten.keys()).sort().map(slug => {
|
||||
const raw_source = rewritten.get(slug);
|
||||
return { slug, source_id: sourceId, ...(raw_source ? { raw_source } : {}) };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1308,12 +1320,12 @@ async function hasLegacySingleChunkCompletion(
|
||||
*/
|
||||
async function stampDreamProvenance(
|
||||
engine: BrainEngine,
|
||||
refs: Array<{ slug: string; source_id: string }>,
|
||||
refs: Array<{ slug: string; source_id: string; raw_source?: string }>,
|
||||
cycleDate: string,
|
||||
): Promise<void> {
|
||||
if (refs.length === 0) return;
|
||||
const { executeRawJsonb } = await import('../sql-query.ts');
|
||||
for (const { slug, source_id } of refs) {
|
||||
for (const { slug, source_id, raw_source } of refs) {
|
||||
try {
|
||||
await executeRawJsonb(
|
||||
engine,
|
||||
@@ -1321,7 +1333,14 @@ async function stampDreamProvenance(
|
||||
SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || $3::jsonb
|
||||
WHERE slug = $1 AND source_id = $2`,
|
||||
[slug, source_id],
|
||||
[{ dream_generated: true, dream_cycle_date: cycleDate }],
|
||||
// #1978 raw-source persistence: record the transcript path the
|
||||
// synthesis was derived from, so `gbrain doctor` (raw_provenance
|
||||
// check) can verify every generated page carries a raw trace.
|
||||
[{
|
||||
dream_generated: true,
|
||||
dream_cycle_date: cycleDate,
|
||||
...(raw_source ? { raw_source } : {}),
|
||||
}],
|
||||
);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
@@ -1423,7 +1442,15 @@ async function writeSummaryPage(
|
||||
// parseMarkdown below round-trips it into the DB-stored frontmatter, so the
|
||||
// marker survives any later reverse-render of the summary page.
|
||||
const fullMarkdown = serializeMarkdown(
|
||||
{ dream_generated: true, dream_cycle_date: summaryDate } as Record<string, unknown>,
|
||||
{
|
||||
dream_generated: true,
|
||||
dream_cycle_date: summaryDate,
|
||||
// #1978: deterministic index page — no source document of its own;
|
||||
// raw traces live on the listed pages. Explicit exemption keeps the
|
||||
// doctor raw_provenance check quiet.
|
||||
raw_trace_exempt: true,
|
||||
raw_trace_exempt_reason: 'deterministic dream-cycle index; raw traces live on listed pages',
|
||||
} as Record<string, unknown>,
|
||||
body,
|
||||
'',
|
||||
{ type: 'note' as string, title: `Dream cycle ${summaryDate}`, tags: ['dream-cycle'] },
|
||||
|
||||
@@ -99,6 +99,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'orphan_ratio',
|
||||
'oversized_pages',
|
||||
'quarantined_pages',
|
||||
'raw_provenance',
|
||||
'flagged_pages',
|
||||
'salience_health',
|
||||
'scraper_junk_pages',
|
||||
|
||||
@@ -71,9 +71,8 @@ export function assertEmbeddingEnabled(cfg: { embedding_disabled?: boolean } | n
|
||||
throw new EmbeddingDisabledError(
|
||||
'This brain was initialized with `--no-embedding` (deferred setup).\n' +
|
||||
'Configure an embedding provider before running embed / import:\n' +
|
||||
' gbrain config set embedding_model <provider>:<model>\n' +
|
||||
' gbrain config set embedding_dimensions <N>\n' +
|
||||
' gbrain init --force --embedding-model <provider>:<model> # re-init to size schema\n',
|
||||
' gbrain init --force --embedding-model <provider>:<model> # re-init to size schema\n' +
|
||||
'(`gbrain config set embedding_model` is refused — schema-sizing fields are set at init.)\n',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,6 +157,11 @@ function buildReceiptFrontmatter(input: ExtractReceiptInput): Record<string, unk
|
||||
const fm: Record<string, unknown> = {
|
||||
type: 'extract_receipt',
|
||||
dream_generated: true,
|
||||
// #1978: receipts record an operation, not a source document — the
|
||||
// run_id/round fields ARE the provenance. Explicit exemption keeps the
|
||||
// doctor raw_provenance check quiet.
|
||||
raw_trace_exempt: true,
|
||||
raw_trace_exempt_reason: 'operation receipt; provenance is run_id + round',
|
||||
kind: input.kind,
|
||||
source_id: input.source_id,
|
||||
run_id: input.run_id,
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
/**
|
||||
* Brainstorm model configurability (takeover of PR #1855 by @starm2010).
|
||||
*
|
||||
* - The cost preview + hard cost ceiling price the model that will actually
|
||||
* run: --model override → configured chat_model → gateway fallback. Before
|
||||
* this, the preview always priced anthropic:claude-sonnet-4-6 even when
|
||||
* the configured chat_model was something else.
|
||||
* - The judge phase honors the `models.brainstorm.judge` config key when no
|
||||
* --judge-model flag is passed.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
resolveBrainstormChatModel,
|
||||
resolveBrainstormJudgeModel,
|
||||
} from '../../src/core/brainstorm/orchestrator.ts';
|
||||
import type { BrainEngine } from '../../src/core/engine.ts';
|
||||
|
||||
function mockEngine(configValues: Record<string, string>): { engine: BrainEngine; reads: string[] } {
|
||||
const reads: string[] = [];
|
||||
const engine = {
|
||||
async getConfig(key: string): Promise<string | null> {
|
||||
reads.push(key);
|
||||
return configValues[key] ?? null;
|
||||
},
|
||||
} as unknown as BrainEngine;
|
||||
return { engine, reads };
|
||||
}
|
||||
|
||||
describe('resolveBrainstormChatModel', () => {
|
||||
test('--model override wins over config', () => {
|
||||
expect(resolveBrainstormChatModel({ chat_model: 'openai:gpt-5' }, 'anthropic:claude-opus-4-6'))
|
||||
.toBe('anthropic:claude-opus-4-6');
|
||||
});
|
||||
|
||||
test('configured chat_model wins over the hardcoded fallback', () => {
|
||||
expect(resolveBrainstormChatModel({ chat_model: 'openai:gpt-5' }))
|
||||
.toBe('openai:gpt-5');
|
||||
});
|
||||
|
||||
test('falls back to the gateway default model when nothing is configured', () => {
|
||||
expect(resolveBrainstormChatModel({})).toBe('anthropic:claude-sonnet-4-6');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveBrainstormJudgeModel', () => {
|
||||
test('--judge-model flag wins without touching config', async () => {
|
||||
const { engine, reads } = mockEngine({ 'models.brainstorm.judge': 'openai:gpt-5' });
|
||||
const out = await resolveBrainstormJudgeModel(engine, 'anthropic:claude-opus-4-6');
|
||||
expect(out).toBe('anthropic:claude-opus-4-6');
|
||||
expect(reads).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('models.brainstorm.judge config key is honored when no flag is passed', async () => {
|
||||
const { engine, reads } = mockEngine({ 'models.brainstorm.judge': 'openai:gpt-5' });
|
||||
const out = await resolveBrainstormJudgeModel(engine);
|
||||
expect(out).toBe('openai:gpt-5');
|
||||
expect(reads).toEqual(['models.brainstorm.judge']);
|
||||
});
|
||||
|
||||
test('returns undefined (defer to modelOverride / gateway default) when unset', async () => {
|
||||
const { engine } = mockEngine({});
|
||||
expect(await resolveBrainstormJudgeModel(engine)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -117,6 +117,22 @@ describe('C6: collectChildPutPageSlugs survives double-encoded jsonb (#745)', ()
|
||||
expect(refs.length).toBeGreaterThan(0);
|
||||
for (const r of refs) expect(r.source_id).toBe('default');
|
||||
});
|
||||
|
||||
// #1978: refs carry the source transcript path when the orchestrator
|
||||
// supplies a job_id → path map, so stampDreamProvenance can persist it.
|
||||
test('stamps refs with raw_source from the jobRawSource map (#1978)', async () => {
|
||||
const jobRawSource = new Map([[1001, '/transcripts/2026-07-01-standup.md']]);
|
||||
const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', jobRawSource);
|
||||
const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
|
||||
expect(ref?.raw_source).toBe('/transcripts/2026-07-01-standup.md');
|
||||
});
|
||||
|
||||
test('omits raw_source when no map entry exists for the job (#1978)', async () => {
|
||||
const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', new Map());
|
||||
const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
|
||||
expect(ref).toBeDefined();
|
||||
expect('raw_source' in (ref as object)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2569: stampDreamProvenance persists the marker into DB frontmatter', () => {
|
||||
@@ -151,4 +167,31 @@ describe('#2569: stampDreamProvenance persists the marker into DB frontmatter',
|
||||
await stampDreamProvenance(engine as any, refs, '2026-07-17'); // no throw
|
||||
await stampDreamProvenance(engine as any, refs, '2026-07-17'); // idempotent
|
||||
});
|
||||
|
||||
// #1978: raw-source persistence — the stamp carries the transcript path
|
||||
// the synthesis was derived from, when the ref supplies one.
|
||||
test('persists raw_source into pages.frontmatter when the ref carries it (#1978)', async () => {
|
||||
await engine.putPage('wiki/originals/ideas/2026-07-17-raw-src-def456', {
|
||||
type: 'note',
|
||||
title: 'Raw source stamp',
|
||||
compiled_truth: 'body',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
await stampDreamProvenance(
|
||||
engine as any,
|
||||
[{
|
||||
slug: 'wiki/originals/ideas/2026-07-17-raw-src-def456',
|
||||
source_id: 'default',
|
||||
raw_source: '/transcripts/2026-07-17-standup.md',
|
||||
}],
|
||||
'2026-07-17',
|
||||
);
|
||||
const rows = await engine.executeRaw<{ fm: Record<string, unknown> }>(
|
||||
`SELECT frontmatter AS fm FROM pages WHERE slug = 'wiki/originals/ideas/2026-07-17-raw-src-def456'`,
|
||||
);
|
||||
const fm = rows[0].fm as Record<string, unknown>;
|
||||
expect(fm.dream_generated).toBe(true);
|
||||
expect(fm.raw_source).toBe('/transcripts/2026-07-17-standup.md');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* #1978 — raw-source persistence guarantee (warn-only v1).
|
||||
*
|
||||
* `rawProvenanceCheck` flags synthesized/derived pages (dream_generated:true
|
||||
* frontmatter or type:synthesis) that carry NO raw trace (raw_trace /
|
||||
* raw_source / source_uri frontmatter, attached raw_data row, or
|
||||
* synthesis_evidence rows) and NO explicit raw_trace_exempt marker.
|
||||
*
|
||||
* Runs against real PGLite so the SQL shape (`?|` key-existence operator +
|
||||
* NOT EXISTS subqueries) is pinned on an actual engine, not a mock.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { rawProvenanceCheck } from '../src/commands/doctor.ts';
|
||||
import { categorizeCheck } from '../src/core/doctor-categories.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
describe('rawProvenanceCheck (#1978, warn-only v1)', () => {
|
||||
test('empty brain → ok', async () => {
|
||||
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
|
||||
expect(result.name).toBe('raw_provenance');
|
||||
expect(result.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('flags only the synthesized page without a trace; every trace/exemption shape passes', async () => {
|
||||
// 1. VIOLATION: dream-generated, no trace, no exemption.
|
||||
await engine.putPage('wiki/derived/no-trace', {
|
||||
type: 'note', title: 'No trace', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: { dream_generated: true },
|
||||
});
|
||||
// 2. OK: dream-generated with raw_source frontmatter.
|
||||
await engine.putPage('wiki/derived/with-raw-source', {
|
||||
type: 'note', title: 'Has raw_source', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: { dream_generated: true, raw_source: '/transcripts/2026-07-01.md' },
|
||||
});
|
||||
// 3. OK: type:synthesis with explicit exemption.
|
||||
await engine.putPage('synthesis/exempt-page', {
|
||||
type: 'synthesis', title: 'Exempt', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: { raw_trace_exempt: true, raw_trace_exempt_reason: 'test' },
|
||||
});
|
||||
// 4. OK: hand-authored note — not synthesized, never flagged.
|
||||
await engine.putPage('wiki/hand-authored', {
|
||||
type: 'note', title: 'Hand authored', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
// 5. OK: dream-generated with an attached raw_data row.
|
||||
const withRaw = await engine.putPage('wiki/derived/with-raw-data', {
|
||||
type: 'note', title: 'Has raw_data', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: { dream_generated: true },
|
||||
});
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO raw_data (page_id, source, data) VALUES ($1, 'test', '{}'::jsonb)`,
|
||||
[withRaw.id],
|
||||
);
|
||||
|
||||
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toContain('1 synthesized page(s)');
|
||||
expect(result.message).toContain('wiki/derived/no-trace');
|
||||
expect(result.message).not.toContain('with-raw-source');
|
||||
expect(result.message).not.toContain('exempt-page');
|
||||
expect(result.message).not.toContain('hand-authored');
|
||||
expect(result.message).not.toContain('with-raw-data');
|
||||
});
|
||||
|
||||
test('stamping an exemption on the violator clears the warning', async () => {
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET frontmatter = frontmatter || '{"raw_trace_exempt": true, "raw_trace_exempt_reason": "reviewed"}'::jsonb
|
||||
WHERE slug = 'wiki/derived/no-trace'`,
|
||||
);
|
||||
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
|
||||
expect(result.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('soft-deleted violators are not flagged', async () => {
|
||||
await engine.putPage('wiki/derived/deleted-no-trace', {
|
||||
type: 'note', title: 'Deleted violator', compiled_truth: 'body', timeline: '',
|
||||
frontmatter: { dream_generated: true },
|
||||
});
|
||||
expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('warn');
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET deleted_at = now() WHERE slug = 'wiki/derived/deleted-no-trace'`,
|
||||
);
|
||||
expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('ok');
|
||||
});
|
||||
|
||||
test('query failure degrades to warn, never throws', async () => {
|
||||
const broken = { executeRaw: async () => { throw new Error('boom'); } } as unknown as BrainEngine;
|
||||
const result = await rawProvenanceCheck(broken);
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toContain('Could not check');
|
||||
});
|
||||
|
||||
test('raw_provenance is categorized as a brain check', () => {
|
||||
expect(categorizeCheck('raw_provenance')).toBe('brain');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* #2301 — re-init with an explicit --embedding-model must recover a brain
|
||||
* that was initialized with --no-embedding (deferred setup).
|
||||
*
|
||||
* Pre-fix: resolveAIOptions honored the persisted `embedding_disabled: true`
|
||||
* sentinel BEFORE the explicit flag and never cleared noEmbedding, and the
|
||||
* persistence merge carried the sentinel forward via ...existingFile. Result:
|
||||
* every re-init (including the recovery command the deferred-setup error
|
||||
* itself recommends) silently re-deferred embedding, forever.
|
||||
*
|
||||
* Hermetic: in-process runInit, GBRAIN_HOME pinned to a tmpdir (same pattern
|
||||
* as test/e2e/fresh-install-pglite.test.ts).
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, readFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
|
||||
|
||||
describe('E2E: re-init with --embedding-model after --no-embedding init (#2301)', () => {
|
||||
let tmpHome: string;
|
||||
let origHome: string | undefined;
|
||||
let origZeKey: string | undefined;
|
||||
let origOpenaiKey: string | undefined;
|
||||
let origVoyageKey: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-e2e-reinit-'));
|
||||
origHome = process.env.GBRAIN_HOME;
|
||||
origZeKey = process.env.ZEROENTROPY_API_KEY;
|
||||
origOpenaiKey = process.env.OPENAI_API_KEY;
|
||||
origVoyageKey = process.env.VOYAGE_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
delete process.env.VOYAGE_API_KEY;
|
||||
process.env.GBRAIN_HOME = tmpHome;
|
||||
process.env.ZEROENTROPY_API_KEY = 'sk-test-ze';
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
if (origHome === undefined) delete process.env.GBRAIN_HOME;
|
||||
else process.env.GBRAIN_HOME = origHome;
|
||||
if (origZeKey === undefined) delete process.env.ZEROENTROPY_API_KEY;
|
||||
else process.env.ZEROENTROPY_API_KEY = origZeKey;
|
||||
if (origOpenaiKey !== undefined) process.env.OPENAI_API_KEY = origOpenaiKey;
|
||||
if (origVoyageKey !== undefined) process.env.VOYAGE_API_KEY = origVoyageKey;
|
||||
// Restore legacy-preload gateway state (mirrors fresh-install-pglite.test.ts).
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { ...process.env },
|
||||
});
|
||||
});
|
||||
|
||||
async function runInitCapturing(args: string[]): Promise<string> {
|
||||
const { runInit } = await import('../../src/commands/init.ts');
|
||||
const origLog = console.log;
|
||||
const origWarn = console.warn;
|
||||
const stdoutBuf: string[] = [];
|
||||
console.log = (...a: unknown[]) => {
|
||||
stdoutBuf.push(a.map(x => (typeof x === 'string' ? x : JSON.stringify(x))).join(' '));
|
||||
};
|
||||
console.warn = () => {};
|
||||
try {
|
||||
await runInit(args);
|
||||
} finally {
|
||||
console.log = origLog;
|
||||
console.warn = origWarn;
|
||||
}
|
||||
return stdoutBuf.join('\n');
|
||||
}
|
||||
|
||||
const cfgPath = () => join(tmpHome, '.gbrain', 'config.json');
|
||||
const readCfg = () => JSON.parse(readFileSync(cfgPath(), 'utf-8'));
|
||||
|
||||
test('explicit --embedding-model clears the persisted embedding_disabled sentinel', async () => {
|
||||
// Step 1: deferred-setup init writes the sentinel.
|
||||
const out1 = await runInitCapturing(['--pglite', '--non-interactive', '--no-embedding']);
|
||||
expect(out1).toContain('deferred setup');
|
||||
const cfg1 = readCfg();
|
||||
expect(cfg1.embedding_disabled).toBe(true);
|
||||
expect(cfg1.embedding_model).toBeUndefined();
|
||||
|
||||
// Step 2: re-init with an explicit embedding model — the recovery path.
|
||||
// Pre-fix this printed the deferred-setup line again and re-persisted
|
||||
// embedding_disabled: true.
|
||||
const out2 = await runInitCapturing([
|
||||
'--pglite', '--non-interactive', '--skip-embed-check',
|
||||
'--embedding-model', 'zeroentropyai:zembed-1',
|
||||
'--embedding-dimensions', '1280',
|
||||
]);
|
||||
expect(out2).not.toContain('deferred setup');
|
||||
expect(out2).toContain('zeroentropyai:zembed-1');
|
||||
|
||||
const cfg2 = readCfg();
|
||||
expect(cfg2.embedding_model).toBe('zeroentropyai:zembed-1');
|
||||
expect(cfg2.embedding_dimensions).toBe(1280);
|
||||
expect(cfg2.embedding_disabled).toBeUndefined();
|
||||
}, 60000);
|
||||
|
||||
test('re-init WITHOUT flags still honors the deferred-setup sentinel (no regression)', async () => {
|
||||
await runInitCapturing(['--pglite', '--non-interactive', '--no-embedding']);
|
||||
const out = await runInitCapturing(['--pglite', '--non-interactive']);
|
||||
expect(out).toContain('deferred setup');
|
||||
const cfg = readCfg();
|
||||
expect(cfg.embedding_disabled).toBe(true);
|
||||
expect(cfg.embedding_model).toBeUndefined();
|
||||
}, 60000);
|
||||
});
|
||||
@@ -115,6 +115,11 @@ describe('writeReceipt — frontmatter D-EXTRACT-19 belt+suspenders', () => {
|
||||
// belt + suspenders: both anti-loop flags are present
|
||||
expect(page.frontmatter?.type).toBe('extract_receipt');
|
||||
expect(page.frontmatter?.dream_generated).toBe(true);
|
||||
// #1978: receipts are operation records, not derived documents —
|
||||
// explicit raw-trace exemption so the doctor raw_provenance check
|
||||
// (warn-only v1) stays quiet.
|
||||
expect(page.frontmatter?.raw_trace_exempt).toBe(true);
|
||||
expect(typeof page.frontmatter?.raw_trace_exempt_reason).toBe('string');
|
||||
});
|
||||
|
||||
test('stamps optional model_id + eval_pass + eval_score when supplied', async () => {
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { withEnv, emptyHome } from './helpers/with-env.ts';
|
||||
import {
|
||||
runPhaseProposeTakes,
|
||||
parseExtractorOutput,
|
||||
@@ -53,14 +52,6 @@ function buildMockEngine(opts: {
|
||||
},
|
||||
async executeRaw<T>(sql: string, params?: unknown[]): Promise<T[]> {
|
||||
captured.push({ sql, params: params ?? [] });
|
||||
// Narrow candidate-page projection (replaces listPages in the phase).
|
||||
if (sql.includes('SELECT slug, source_id, compiled_truth')) {
|
||||
return opts.pages.map((p) => ({
|
||||
slug: p.slug,
|
||||
source_id: p.source_id,
|
||||
compiled_truth: p.compiled_truth,
|
||||
})) as T[];
|
||||
}
|
||||
// SELECT idempotency check
|
||||
if (sql.includes('SELECT id FROM take_proposals')) {
|
||||
const [sourceId, slug, ch, pv] = params ?? [];
|
||||
@@ -485,72 +476,4 @@ New prose appended here.`;
|
||||
resetGateway();
|
||||
}
|
||||
});
|
||||
|
||||
test('default extractor skips cleanly when the Anthropic chat model has no key', async () => {
|
||||
// Empty GBRAIN_HOME so hasAnthropicKey's config-file fallback can't find
|
||||
// the operator's real key.
|
||||
await withEnv({ GBRAIN_HOME: emptyHome(), ANTHROPIC_API_KEY: undefined }, async () => {
|
||||
configureGateway({ chat_model: 'anthropic:claude-sonnet-4-6', env: {} });
|
||||
try {
|
||||
const { engine, captured } = buildMockEngine({
|
||||
pages: [buildPage({ slug: 'wiki/a', body: 'claim-ish prose' })],
|
||||
});
|
||||
const result = await runPhaseProposeTakes(buildCtx(engine));
|
||||
|
||||
expect(result.status).toBe('skipped');
|
||||
expect((result.details as Record<string, unknown>).reason).toBe('no_provider');
|
||||
// Skips BEFORE touching the engine — no page scan, no cache probes.
|
||||
expect(captured).toHaveLength(0);
|
||||
} finally {
|
||||
resetGateway();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('an injected extractor is never gated on provider availability', async () => {
|
||||
await withEnv({ GBRAIN_HOME: emptyHome(), ANTHROPIC_API_KEY: undefined }, async () => {
|
||||
configureGateway({ chat_model: 'anthropic:claude-sonnet-4-6', env: {} });
|
||||
try {
|
||||
const { engine } = buildMockEngine({
|
||||
pages: [buildPage({ slug: 'wiki/b', body: 'still processed' })],
|
||||
});
|
||||
const extractor: ProposeTakesExtractor = async () => [];
|
||||
const result = await runPhaseProposeTakes(buildCtx(engine), { extractor });
|
||||
|
||||
expect(result.status).toBe('ok');
|
||||
expect((result.details as Record<string, unknown>).pages_scanned).toBe(1);
|
||||
} finally {
|
||||
resetGateway();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('loads proposal candidates with a narrow page projection', async () => {
|
||||
const pages = [buildPage({ slug: 'wiki/narrow', body: 'A narrow projection avoids unrelated page columns.' })];
|
||||
const { engine, captured } = buildMockEngine({ pages });
|
||||
const extractor: ProposeTakesExtractor = async () => [];
|
||||
await runPhaseProposeTakes(buildCtx(engine), { extractor });
|
||||
|
||||
const pageSelect = captured.find(c => c.sql.includes('FROM pages'));
|
||||
expect(pageSelect).toBeDefined();
|
||||
expect(pageSelect!.sql).toContain('SELECT slug, source_id, compiled_truth');
|
||||
expect(pageSelect!.sql).not.toContain('*');
|
||||
// Scalar sourceId scope from ctx binds as a plain equality param.
|
||||
expect(pageSelect!.params[0]).toBe('default');
|
||||
});
|
||||
|
||||
test('narrow projection: federated sourceIds beat scalar sourceId', async () => {
|
||||
const { engine, captured } = buildMockEngine({ pages: [] });
|
||||
const extractor: ProposeTakesExtractor = async () => [];
|
||||
const ctx = {
|
||||
...buildCtx(engine),
|
||||
auth: { allowedSources: ['team-a', 'team-b'] },
|
||||
} as OperationContext;
|
||||
await runPhaseProposeTakes(ctx, { extractor });
|
||||
|
||||
const pageSelect = captured.find(c => c.sql.includes('FROM pages'));
|
||||
expect(pageSelect).toBeDefined();
|
||||
expect(pageSelect!.sql).toContain('source_id = ANY(');
|
||||
expect(pageSelect!.params[0]).toEqual(['team-a', 'team-b']);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user