Compare commits

..
Author SHA1 Message Date
fffa779172 fix(rerank): classify missing auth before fallback (#2059)
Missing ZEROENTROPY_API_KEY threw AIConfigError from auth resolution, which
rerank.ts recorded as reason 'unknown' — and doctor's reranker_health had no
unknown bucket, so it reported ok while every rerank silently failed open.

- gateway.rerank wraps AIConfigError from applyResolveAuth as
  RerankError(reason: 'auth') before any HTTP call.
- checkRerankerHealth warns on >=3 'unknown' failures in the 7-day window
  (covers historical pre-fix audit rows), with a ZEROENTROPY_API_KEY setup
  hint when the error summary points at a missing key.
- Tests: RerankError(auth) classification, applyReranker fail-open + audit
  reason, doctor warn on repeated unknowns.

Takeover of #2070.

Co-authored-by: maxpetrusenkoagent <maxpetrusenkoagent@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:43:03 -07:00
16 changed files with 131 additions and 165 deletions
+18
View File
@@ -1551,6 +1551,24 @@ export async function checkRerankerHealth(engine: BrainEngine): Promise<Check> {
};
}
// Historical #2059 rows were logged as `unknown` before missing reranker
// auth was classified at the gateway. Surface repeated unknowns instead of
// reporting "ok" while every rerank fails open.
const unknownFails = failures.filter((f) => f.reason === 'unknown');
if (unknownFails.length >= 3) {
const setupHint = unknownFails.some((f) => {
const summary = String(f.error_summary ?? '');
return summary.includes('ZEROENTROPY_API_KEY') || summary.toLowerCase().includes('api key');
})
? ' Fix: verify ZEROENTROPY_API_KEY and run `gbrain models doctor`.'
: '';
return {
name: 'reranker_health',
status: 'warn',
message: `${unknownFails.length} unknown reranker failure(s) in last 7 days.${setupHint}`,
};
}
return {
name: 'reranker_health',
status: 'ok',
+9 -1
View File
@@ -3656,7 +3656,15 @@ export async function rerank(input: RerankInput): Promise<RerankResult[]> {
// whose request/response shape differs from ZE/llama.cpp (e.g. Voyage with
// `top_k` / `data[]`) needs separate adapter hooks in a follow-up plan.
const url = `${compat.baseURL.replace(/\/$/, '')}${tp.path ?? '/models/rerank'}`;
const auth = applyResolveAuth(recipe, cfg, 'reranker');
let auth: { apiKey?: string; headers?: Record<string, string> };
try {
auth = applyResolveAuth(recipe, cfg, 'reranker');
} catch (err) {
if (err instanceof AIConfigError) {
throw new RerankError(err.message, 'auth');
}
throw err;
}
// applyResolveAuth returns { apiKey } for Bearer-style auth (SDK's native
// path) or { headers } for custom-header providers (Azure). v0.37.6.0:
// recipes can ALSO declare default_headers (attribution etc.) which flow
-4
View File
@@ -908,10 +908,6 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
'facts.extraction_model',
// #2113: output-token cap for the per-turn facts extractor (default 4000).
'facts.extraction_max_tokens',
// Owner opt-in: let the local stdio MCP pipe read this owner's private facts
// (find_trajectory / recall). Default off; HTTP transport ignores it. See
// src/core/facts/reader-trust.ts.
'facts.trust_local_reads',
// Dream cycle config
'dream.synthesize.session_corpus_dir',
'dream.synthesize.meeting_transcripts_dir',
-2
View File
@@ -591,8 +591,6 @@ export interface TrajectoryOpts {
sourceIds?: string[];
/** When true, filters to visibility='world' only. Set by MCP layer from ctx.remote. */
remote?: boolean;
/** Owner opt-in: read private facts despite `remote`. See facts/reader-trust.ts. */
trustedFactReads?: boolean;
/** Metric filter. When set, only facts with this canonical metric label participate. */
metric?: string;
/**
+5 -8
View File
@@ -17,7 +17,6 @@
import type { OperationContext } from './../operations.ts';
import type { FactRow } from './../engine.ts';
import { effectiveConfidence } from './decay.ts';
import { readableFactVisibilities } from './reader-trust.ts';
const DEFAULT_TTL_MS = 30_000;
const DEFAULT_TOP_K = 10;
@@ -51,13 +50,7 @@ export async function getBrainHotMemoryMeta(
const sessionId = (ctx as { source_session?: string }).source_session
?? null;
const allowListHash = hashAllowList(ctx.takesHoldersAllowList);
// Visibility tier: untrusted remote → world-only; trusted local +
// owner-trusted reads → all rows. Folded into the cache key (the header's
// "cache entries don't bleed across tiers" invariant): trustedFactReads is
// re-read from config per call, so a mid-session opt-out must not keep
// serving a private-inclusive cached payload for the TTL window.
const visibility = readableFactVisibilities(ctx);
const cacheKey = `${sourceId}::${sessionId ?? '_'}::${allowListHash}::${visibility ? 'world' : 'all'}`;
const cacheKey = `${sourceId}::${sessionId ?? '_'}::${allowListHash}`;
const ttl = Math.max(1000, opts.ttlMs ?? DEFAULT_TTL_MS);
const topK = Math.max(1, Math.min(opts.topK ?? DEFAULT_TOP_K, 25));
@@ -68,6 +61,10 @@ export async function getBrainHotMemoryMeta(
return cached.payload;
}
// Build a fresh payload. Visibility tier: remote → world-only;
// local → all rows.
const visibility = ctx.remote === false ? undefined : ['world'] as ('world' | 'private')[];
let rows: FactRow[] = [];
if (sessionId) {
rows = await ctx.engine.listFactsBySession(sourceId, sessionId, {
-48
View File
@@ -1,48 +0,0 @@
/**
* Fact-read visibility trust.
*
* Fact rows are tagged `private` | `world`. Remote/untrusted callers
* (`remote === true`) see only `world` rows — the posture that keeps a
* published or HTTP-served brain from leaking private claims to strangers.
*
* But the stdio MCP server is an unauthenticated LOCAL pipe: on a single-owner
* machine the caller IS the owner, yet it still defaults `remote: true` for
* safety, so the owner's own agent is denied the owner's own private facts
* (e.g. `find_trajectory` returns empty over MCP even though the facts exist).
*
* `trustedFactReads` is a narrow, READ-ONLY trust elevation, deliberately
* DECOUPLED from `remote` so every other remote protection — file_upload
* confinement, source isolation, fence stripping, takes-holder scoping — stays
* fully in force. The stdio MCP server sets it ONLY when the brain owner opts
* in via the `facts.trust_local_reads` config (default off). The HTTP/published
* transport never sets it, so a served brain stays world-only regardless.
*/
export interface FactReaderTrust {
/**
* Mirrors OperationContext.remote. FAIL-CLOSED: anything not strictly
* `false` is treated as remote/untrusted (CLAUDE.md trust invariant).
*/
remote?: boolean;
/** Owner opt-in: this remote caller may read private facts. */
trustedFactReads?: boolean;
}
/**
* True when the reader is restricted to `visibility = 'world'` rows.
* Fail-closed: an unset/undefined `remote` is untrusted — only an explicit
* `remote: false` (trusted local CLI) or an explicit owner opt-in
* (`trustedFactReads: true`) reads private rows.
*/
export function factsWorldOnly(t: FactReaderTrust): boolean {
return t.remote !== false && t.trustedFactReads !== true;
}
/**
* Visibility filter for list-style fact reads: `['world']` when the reader is
* world-only, `undefined` (no filter — all rows) when it is trusted.
*/
export function readableFactVisibilities(
t: FactReaderTrust,
): ('private' | 'world')[] | undefined {
return factsWorldOnly(t) ? ['world'] : undefined;
}
+7 -14
View File
@@ -18,7 +18,6 @@ import { captureEvalCandidate, isEvalCaptureEnabled, isEvalScrubEnabled } from '
import type { HybridSearchMeta } from './types.ts';
import { extractPageLinks, isAutoLinkEnabled, isAutoTimelineEnabled, isGlobalBasenameEnabled, parseTimelineEntries, makeResolver, type UnresolvedFrontmatterRef } from './link-extraction.ts';
import { isFactsBackstopEligible } from './facts/eligibility.ts';
import { readableFactVisibilities } from './facts/reader-trust.ts';
import { stripTakesFence } from './takes-fence.ts';
import { stripFactsFence } from './facts-fence.ts';
import { getContentFlag } from './quarantine.ts';
@@ -333,15 +332,6 @@ export interface OperationContext {
* remote/untrusted (defense in depth in case the type is bypassed via cast).
*/
remote: boolean;
/**
* Owner opt-in (`facts.trust_local_reads`): allow this remote caller to read
* `private` facts. A NARROW, read-only trust elevation decoupled from
* `remote` — every other remote protection (file confinement, source
* isolation, fence stripping, takes scoping) stays in force. Set ONLY by the
* stdio MCP server when the config is on; the HTTP transport never sets it.
* Consulted via `src/core/facts/reader-trust.ts`.
*/
trustedFactReads?: boolean;
/**
* Subagent runtime context (v0.16+). Set by the subagent tool dispatcher when
* dispatching an op as a tool call from an LLM loop. Used to enforce per-op
@@ -3639,7 +3629,6 @@ const find_trajectory: Operation = {
entitySlug: p.entity_slug,
...scope,
remote: ctx.remote === true,
trustedFactReads: ctx.trustedFactReads === true,
metric,
kind,
since,
@@ -4000,9 +3989,13 @@ const recall: Operation = {
const includeExpired = p.include_expired === true;
const grep = typeof p.grep === 'string' ? p.grep.toLowerCase() : null;
// Visibility filter: world-only for untrusted remote callers; all rows for
// trusted local CLI and owner-trusted reads (facts.trust_local_reads).
const visibility = readableFactVisibilities(ctx);
// Visibility filter: remote callers see world-only unless their token
// grants elevated visibility (future-proofing; v0.31 ships world-only
// for remote, all for local CLI).
const visibility =
ctx.remote === false
? undefined
: ['world'] as ('private' | 'world')[];
let rows: Awaited<ReturnType<typeof ctx.engine.listFactsByEntity>> = [];
+1 -5
View File
@@ -18,7 +18,6 @@ import type {
} from './engine.ts';
import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts';
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
import { factsWorldOnly } from './facts/reader-trust.ts';
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
import { runMigrations } from './migrate.ts';
import { PGLITE_SCHEMA_SQL, getPGLiteSchema } from './pglite-schema.ts';
@@ -4319,10 +4318,7 @@ export class PGLiteEngine implements BrainEngine {
const useArray = Array.isArray(opts.sourceIds) && opts.sourceIds.length > 0;
const sourceIds = useArray ? opts.sourceIds! : null;
const sourceId = opts.sourceId ?? 'default';
// Direct-engine contract: unset `remote` here means a trusted in-process
// caller (CLI/tests) — the fail-closed default lives in the op layer, which
// always passes explicit booleans. Normalize before the fail-closed helper.
const remoteFilter = factsWorldOnly({ remote: opts.remote === true, trustedFactReads: opts.trustedFactReads === true });
const remoteFilter = opts.remote === true;
// Build SQL dynamically. PGLite uses $N positional params; we
// assemble the WHERE clauses + params array in tandem to keep them
+1 -5
View File
@@ -14,7 +14,6 @@ import type {
SourceRow,
} from './engine.ts';
import { withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay, type BatchAuditSite } from './retry.ts';
import { factsWorldOnly } from './facts/reader-trust.ts';
import { logBatchRetry as auditLogBatchRetry, logBatchExhausted as auditLogBatchExhausted } from './audit/batch-retry-audit.ts';
import type {
DomainBankSampleOpts, CorpusSampleOpts, DomainBankRow,
@@ -4533,10 +4532,7 @@ export class PostgresEngine implements BrainEngine {
const useArray = Array.isArray(opts.sourceIds) && opts.sourceIds.length > 0;
const sourceIds = useArray ? opts.sourceIds! : null;
const sourceId = opts.sourceId ?? 'default';
// Direct-engine contract: unset `remote` here means a trusted in-process
// caller (CLI/tests) — the fail-closed default lives in the op layer, which
// always passes explicit booleans. Normalize before the fail-closed helper.
const remoteFilter = factsWorldOnly({ remote: opts.remote === true, trustedFactReads: opts.trustedFactReads === true });
const remoteFilter = opts.remote === true;
// Source-scope predicate: array path (federated) wins over scalar.
// Engine.ts contract: returns chronological points; regressions +
-8
View File
@@ -30,13 +30,6 @@ export interface ToolResult {
export interface DispatchOpts {
/** Defaults to true (remote/untrusted). Local CLI callers (`gbrain call`) pass false. */
remote?: boolean;
/**
* Owner opt-in (`facts.trust_local_reads`): let this remote caller read
* `private` facts. Set ONLY by the stdio MCP server; the HTTP transport
* leaves it unset so a served brain stays world-only. See
* `src/core/facts/reader-trust.ts`.
*/
trustedFactReads?: boolean;
/** Override the default stderr logger (e.g. CLI uses console.* directly). */
logger?: OperationContext['logger'];
/**
@@ -210,7 +203,6 @@ export function buildOperationContext(
logger: opts.logger || stderrLogger,
dryRun: !!params.dry_run,
remote: opts.remote ?? true,
trustedFactReads: opts.trustedFactReads === true,
takesHoldersAllowList: opts.takesHoldersAllowList,
// v0.34 D4: sourceId is REQUIRED at the type level. Auto-fill 'default'
// for single-source brains and any caller who didn't resolve a sourceId.
-11
View File
@@ -35,16 +35,6 @@ export async function startMcpServer(engine: BrainEngine) {
// shape and cast through `any` (the SDK accepts it via the ServerResult union).
server.setRequestHandler(CallToolRequestSchema, async (request: any): Promise<any> => {
const { name, arguments: params } = request.params;
// Owner opt-in: the stdio pipe is local + unauthenticated, so on a
// single-owner machine its caller is the owner. When facts.trust_local_reads
// is on, let fact reads (find_trajectory / recall) see this owner's own
// private facts. Narrow + read-only — every other remote protection stays
// on (remote stays true). HTTP transport never sets this. Best-effort: a
// config read blip falls back to the safe world-only default.
let trustedFactReads = false;
try {
trustedFactReads = (await engine.getConfig('facts.trust_local_reads')) === 'true';
} catch { /* keep world-only default */ }
// v0.28: stdio MCP has no per-token auth (local pipe). Default the
// takes-holder allow-list to ['world'] so agent-facing callers don't
// see private hunches via takes_list / takes_search / query. Operators
@@ -52,7 +42,6 @@ export async function startMcpServer(engine: BrainEngine) {
// `gbrain call <op>` (sets remote=false in src/cli.ts).
return dispatchToolCall(engine, name, params, {
remote: true,
trustedFactReads,
takesHoldersAllowList: ['world'],
// v0.31: source defaults to 'default' for stdio (no per-token scope).
// Operators who want a different source on stdio MCP should set
+22
View File
@@ -154,6 +154,28 @@ describe('gateway.rerank() — happy path', () => {
describe('gateway.rerank() — error classification', () => {
beforeEach(() => configureZE());
test('missing required reranker API key → RerankError(auth) before HTTP call', async () => {
configureGateway({
reranker_model: 'zeroentropyai:zerank-2',
env: {},
});
let called = false;
__setRerankTransportForTests(async () => {
called = true;
return mockResp({ results: [{ index: 0, relevance_score: 0.5 }] });
});
try {
await rerank({ query: 'q', documents: ['d'] });
throw new Error('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(RerankError);
expect((err as RerankError).reason).toBe('auth');
expect((err as Error).message).toContain('ZEROENTROPY_API_KEY');
expect(called).toBe(false);
}
});
test('401 → auth', async () => {
__setRerankTransportForTests(async () => new Response('Unauthorized', { status: 401 }));
try {
+33
View File
@@ -2,6 +2,11 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:tes
import { mkdirSync, rmSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { withEnv } from './helpers/with-env.ts';
import { logRerankFailure } from '../src/core/rerank-audit.ts';
describe('doctor command', () => {
test('doctor module exports runDoctor', async () => {
@@ -47,6 +52,34 @@ describe('doctor command', () => {
expect(check.issues![0].action).toContain('trigger');
});
test('reranker_health warns on repeated unknown rerank failures', async () => {
const { checkRerankerHealth } = await import('../src/commands/doctor.ts');
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-rerank-doctor-'));
try {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
for (let i = 0; i < 3; i++) {
logRerankFailure({
model: 'zeroentropyai:zerank-2',
reason: 'unknown',
query_hash: `unknown${i}`,
doc_count: 30,
error_summary: 'ZeroEntropy reranker requires ZEROENTROPY_API_KEY.',
});
}
const check = await checkRerankerHealth({
async getConfig(key: string): Promise<string | null> {
return key === 'search.reranker.enabled' ? 'true' : null;
},
} as any);
expect(check.status).toBe('warn');
expect(check.message).toContain('unknown');
expect(check.message).toContain('ZEROENTROPY_API_KEY');
});
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
test('runDoctor accepts null engine for filesystem-only mode', async () => {
const { runDoctor } = await import('../src/commands/doctor.ts');
// runDoctor should accept null engine — it runs filesystem checks only.
-16
View File
@@ -165,22 +165,6 @@ describe('findTrajectory — visibility filter (D-CDX-1 / R6)', () => {
const all = await engine.findTrajectory({ entitySlug: 'traj-vis-default' });
expect(all.length).toBe(2);
});
test('remote=true + trustedFactReads bypasses world-only (owner-trusted reads)', async () => {
await insertTyped({ entity_slug: 'traj-vis-trusted', metric: 'mrr', value: 50000, visibility: 'private', valid_from: new Date('2026-01-15') });
await insertTyped({ entity_slug: 'traj-vis-trusted', metric: 'mrr', value: 99999, visibility: 'world', valid_from: new Date('2026-04-12') });
// Untrusted remote: world only.
const untrusted = await engine.findTrajectory({ entitySlug: 'traj-vis-trusted', remote: true });
expect(untrusted.length).toBe(1);
expect(untrusted[0].value).toBe(99999);
// Owner-trusted remote: sees the private point too. remote stays true — only
// fact-read visibility is elevated.
const trusted = await engine.findTrajectory({ entitySlug: 'traj-vis-trusted', remote: true, trustedFactReads: true });
expect(trusted.length).toBe(2);
expect(trusted.map(p => p.value).sort((a, b) => (a! - b!))).toEqual([50000, 99999]);
});
});
describe('findTrajectory — metric + since + until filters', () => {
-43
View File
@@ -1,43 +0,0 @@
import { describe, test, expect } from 'bun:test';
import {
factsWorldOnly,
readableFactVisibilities,
} from '../src/core/facts/reader-trust.ts';
describe('factsWorldOnly', () => {
test('FAIL-CLOSED: unset remote is untrusted (world-only)', () => {
expect(factsWorldOnly({})).toBe(true);
expect(factsWorldOnly({ remote: undefined })).toBe(true);
expect(factsWorldOnly({ trustedFactReads: false })).toBe(true);
});
test('explicit remote=false (trusted local CLI) sees all', () => {
expect(factsWorldOnly({ remote: false })).toBe(false);
expect(factsWorldOnly({ remote: false, trustedFactReads: false })).toBe(false);
});
test('untrusted remote callers are world-only', () => {
expect(factsWorldOnly({ remote: true })).toBe(true);
expect(factsWorldOnly({ remote: true, trustedFactReads: false })).toBe(true);
});
test('owner-trusted remote reads bypass the world-only filter', () => {
expect(factsWorldOnly({ remote: true, trustedFactReads: true })).toBe(false);
});
test('trustedFactReads is a no-op for an already-trusted local caller', () => {
expect(factsWorldOnly({ remote: false, trustedFactReads: true })).toBe(false);
});
});
describe('readableFactVisibilities', () => {
test("world-only readers get the ['world'] filter (incl. unset remote)", () => {
expect(readableFactVisibilities({ remote: true })).toEqual(['world']);
expect(readableFactVisibilities({})).toEqual(['world']);
});
test('trusted readers get undefined (no filter — all rows)', () => {
expect(readableFactVisibilities({ remote: false })).toBeUndefined();
expect(readableFactVisibilities({ remote: true, trustedFactReads: true })).toBeUndefined();
});
});
+35
View File
@@ -12,9 +12,14 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { applyReranker, type RerankerOpts } from '../../src/core/search/rerank.ts';
import { RerankError, type RerankResult } from '../../src/core/ai/gateway.ts';
import { readRecentRerankFailures } from '../../src/core/rerank-audit.ts';
import type { SearchResult } from '../../src/core/types.ts';
import { withEnv } from '../helpers/with-env.ts';
function makeResult(slug: string, score: number, chunk: string): SearchResult {
return {
@@ -160,6 +165,36 @@ describe('applyReranker — fail-open on every RerankError reason', () => {
expect(out).toEqual(results);
});
test('missing gateway reranker API key fail-opens and audits auth', async () => {
const { configureGateway } = await import('../../src/core/ai/gateway.ts');
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-rerank-search-'));
try {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
configureGateway({
reranker_model: 'zeroentropyai:zerank-2',
env: {},
});
const results = [makeResult('a', 1.0, 'doc a')];
const out = await applyReranker('q', results, {
enabled: true,
topNIn: 1,
topNOut: null,
model: 'zeroentropyai:zerank-2',
});
expect(out).toEqual(results);
const failures = readRecentRerankFailures(1);
expect(failures).toHaveLength(1);
expect(failures[0]!.reason).toBe('auth');
expect(failures[0]!.error_summary).toContain('ZEROENTROPY_API_KEY');
});
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
configureGateway({ env: { ZEROENTROPY_API_KEY: 'test-key' } });
}
});
test('fail-open on non-RerankError throw too', async () => {
const results = [makeResult('a', 1.0, 'a')];
const opts: RerankerOpts = {