Compare commits

..
Author SHA1 Message Date
f68eb9d905 fix(takes): kill propose_takes rescan loop, keep every claim, and route cycle/brainstorm models through config (#2804 #2805)
Takeover of #2804 and #2805 (stacked), rebased onto current master.

Rescan loop + dropped claims (#2804):
- Zero-claim scans now write a status='empty' sentinel row so the
  (source_id, page_slug, content_hash, prompt_version) cache hits on
  every subsequent cycle instead of re-spending the extractor LLM call
  on the same unchanged page forever.
- The idempotency index gains md5(claim_text): multi-claim pages keep
  every claim (the 4-column unique index silently dropped claims 2..N
  via ON CONFLICT DO NOTHING). proposals_inserted now counts rows that
  actually landed (RETURNING id), not insert attempts.
- Migration renumbered 123 -> 125 (master shipped v123
  configurable_fts_language and v124 after the PR was cut; the
  duplicate v123 would never have run under the version > current
  runner). Schema parity across schema.sql, pglite-schema.ts,
  schema-embedded.ts.

Model-tier honoring (#2805), reworked for current master:
- propose_takes, grade_takes, calibration_profile resolve their model
  through resolveModel(models.<phase> > models.default > env >
  getChatModel()) — per-phase config keys are now honored while the
  gateway chat model (already tier-resolved by
  reconfigureGatewayWithEngine) stays the default, preserving master's
  #2451 semantics (provider-prefixed stored ids, nested prefixes like
  openrouter:... intact) and the #2997 opts.model ?? getChatModel()
  idiom. The PR's original bare-tail-everywhere approach was dropped
  where it conflicted with #2451; grade_takes keeps the bare tail ONLY
  for its cache key / stored judge_model_id (continuity with historical
  rows), while the judge call itself gets the full resolved string —
  fixing the live label-vs-actual mismatch (call rode chat_model,
  telemetry recorded a hardcoded claude-sonnet-4-6).
- brainstorm resolves via models.brainstorm > tier chain (was
  hardcoded anthropic:claude-sonnet-4-6).
- gbrain models lists the four new per-task keys; output caps and
  context-window entries added for newer Claude models.

Tests: test/propose-takes-rescan.test.ts (new, hermetic PGLite);
config-key coverage in propose-takes/grade-takes tests; existing #2451
and gateway-chat-model tests pass unchanged.

Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:33:19 -07:00
27 changed files with 415 additions and 409 deletions
-59
View File
@@ -820,10 +820,6 @@ export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorRep
// v0.42.x (#1794, 4A): pool-budget nudge when GBRAIN_MAX_CONNECTIONS is set.
checks.push(await checkPoolBudget(engine));
// #2552: warn when an explicit embed-concurrency override fans out against
// a local single-slot embedding endpoint (silent backfill starvation).
checks.push(await checkEmbedConcurrency());
// v0.42.7 (#1696): link-extraction lag. Strictly SQL (single indexed COUNT),
// safe on the thin-client/remote path — remote operators on checkout-less
// Postgres brains are exactly who can't otherwise see the extraction backlog.
@@ -3819,61 +3815,6 @@ export function computePoolBudgetCheck(
};
}
/**
* #2552: warn when an explicit GBRAIN_EMBED_CONCURRENCY override fans out
* against a local single-slot embedding endpoint (Ollama / llama-server /
* localhost base URL). Requests serialize on the one loaded model, so N
* parallel pages multiply latency xN and can exceed the fetch timeout with
* no surfaced error the backfill silently starves. (When the env var is
* unset, embed auto-caps at LOCAL_EMBED_CONCURRENCY_CAP and this check
* reports ok.) Pure; exported for tests.
*/
export function computeEmbedConcurrencyCheck(
isLocalEndpoint: boolean,
envValue: string | undefined,
localCap: number,
): Check {
const name = 'embed_concurrency';
if (!isLocalEndpoint) {
return { name, status: 'ok', message: 'Embedding endpoint is not a local inference server — cloud concurrency defaults apply.' };
}
const parsed = envValue ? parseInt(envValue, 10) : NaN;
if (envValue && Number.isFinite(parsed) && parsed > localCap) {
return {
name,
status: 'warn',
message:
`GBRAIN_EMBED_CONCURRENCY=${parsed} against a local embedding endpoint. ` +
`Local inference servers serialize requests, so ${parsed} parallel pages multiply ` +
`latency x${parsed} and can exceed the fetch timeout — the embed backfill stalls ` +
`with no error. Unset GBRAIN_EMBED_CONCURRENCY (auto-caps at ${localCap}) or set it <= ${localCap}.`,
};
}
return {
name,
status: 'ok',
message: `Local embedding endpoint detected; embed concurrency capped at ${envValue ? parsed : localCap}.`,
};
}
/** Thin gateway/env wrapper over `computeEmbedConcurrencyCheck`. */
export async function checkEmbedConcurrency(): Promise<Check> {
try {
const { isLocalEmbeddingEndpoint, LOCAL_EMBED_CONCURRENCY_CAP } = await import('../core/ai/gateway.ts');
return computeEmbedConcurrencyCheck(
isLocalEmbeddingEndpoint(),
process.env.GBRAIN_EMBED_CONCURRENCY,
LOCAL_EMBED_CONCURRENCY_CAP,
);
} catch (err) {
return {
name: 'embed_concurrency',
status: 'ok',
message: `Skipped (${err instanceof Error ? err.message : String(err)})`,
};
}
}
/** Thin env/engine wrapper over `computePoolBudgetCheck`. */
export async function checkPoolBudget(_engine: BrainEngine): Promise<Check> {
try {
+8 -30
View File
@@ -1,6 +1,5 @@
import type { BrainEngine } from '../core/engine.ts';
import { embedBatch, currentEmbeddingSignature } from '../core/embedding.ts';
import { isLocalEmbeddingEndpoint, LOCAL_EMBED_CONCURRENCY_CAP } from '../core/ai/gateway.ts';
import type { ChunkInput } from '../core/types.ts';
import { chunkText } from '../core/chunkers/recursive.ts';
import { createProgress, type ProgressReporter } from '../core/progress.ts';
@@ -177,31 +176,6 @@ export class EmbeddingDimMismatchError extends Error {
}
}
/**
* #2552: resolve the bulk-embed worker count. Env override or the
* cloud-tuned default of 20 — but when the operator did NOT set
* GBRAIN_EMBED_CONCURRENCY and the embedding endpoint is a local inference
* server (Ollama / llama-server / localhost base URL), cap at
* LOCAL_EMBED_CONCURRENCY_CAP: 20 parallel pages against a single-slot
* server serialize on the one loaded model, multiply latency x20 past the
* fetch timeout, and starve the backfill with no surfaced error. An
* explicit env value always wins (`gbrain doctor` warns instead).
* Pacing only ever LOWERS concurrency (Codex P2).
*/
export function resolveEmbedConcurrency(paceMaxConcurrency?: number): number {
const envSet = !!process.env.GBRAIN_EMBED_CONCURRENCY;
const base = parseInt(process.env.GBRAIN_EMBED_CONCURRENCY || '20', 10);
let resolved = base;
if (!envSet && isLocalEmbeddingEndpoint() && base > LOCAL_EMBED_CONCURRENCY_CAP) {
resolved = LOCAL_EMBED_CONCURRENCY_CAP;
serr(
`[embed] local embedding endpoint detected — capping concurrency at ` +
`${LOCAL_EMBED_CONCURRENCY_CAP} (set GBRAIN_EMBED_CONCURRENCY to override)`,
);
}
return paceMaxConcurrency ? Math.min(resolved, paceMaxConcurrency) : resolved;
}
/**
* Pre-flight check: read the actual schema column dim and compare to the
* gateway's resolved dim. Throws `EmbeddingDimMismatchError` on mismatch
@@ -703,8 +677,10 @@ async function embedAll(
// Paced runs lower this to the resolved cap (the real lever vs pooler-slot
// starvation); unpaced keeps the env/default 20. Codex P2: only ever LOWER —
// never raise above an operator's existing env cap.
// #2552: local endpoints auto-cap — see resolveEmbedConcurrency.
const CONCURRENCY = resolveEmbedConcurrency(staleOpts?.paceMaxConcurrency);
const BASE_CONCURRENCY = parseInt(process.env.GBRAIN_EMBED_CONCURRENCY || '20', 10);
const CONCURRENCY = staleOpts?.paceMaxConcurrency
? Math.min(BASE_CONCURRENCY, staleOpts.paceMaxConcurrency)
: BASE_CONCURRENCY;
async function embedOnePage(page: typeof pages[number]) {
// #1737: bail before doing any work for this page if the run was aborted.
@@ -879,8 +855,10 @@ async function embedAllStale(
// Paced runs lower concurrency to the resolved cap (E-1: worker count IS the
// lever on this single pool, no separate permit). Codex P2: pacing only ever
// LOWERS concurrency — never raise above an operator's existing env cap.
// #2552: local endpoints auto-cap — see resolveEmbedConcurrency.
const CONCURRENCY = resolveEmbedConcurrency(staleOpts?.paceMaxConcurrency);
const BASE_CONCURRENCY = parseInt(process.env.GBRAIN_EMBED_CONCURRENCY || '20', 10);
const CONCURRENCY = staleOpts?.paceMaxConcurrency
? Math.min(BASE_CONCURRENCY, staleOpts.paceMaxConcurrency)
: BASE_CONCURRENCY;
const pacer = staleOpts?.pacer ?? createNoopPacer();
// D3 + D3a + D8: wall-clock budget. 30 min default; env override.
+4
View File
@@ -50,6 +50,10 @@ const PER_TASK_KEYS: Array<{ key: string; tier: ModelTier; description: string }
{ key: 'models.eval.contradictions_judge', tier: 'utility', description: 'Contradiction probe judge (v0.34 temporal-aware)' },
{ key: 'models.expansion', tier: 'utility', description: 'Query expansion for hybrid search' },
{ key: 'models.chat', tier: 'reasoning', description: 'Default `gateway.chat()` model' },
{ key: 'models.propose_takes', tier: 'reasoning', description: 'propose_takes claim extractor' },
{ key: 'models.grade_takes', tier: 'reasoning', description: 'grade_takes verdict judge' },
{ key: 'models.calibration_profile', tier: 'reasoning', description: 'Calibration profile generator' },
{ key: 'models.brainstorm', tier: 'reasoning', description: '`gbrain brainstorm` orchestrator' },
];
interface ModelEntry {
-27
View File
@@ -683,33 +683,6 @@ export function getEmbeddingDimensions(): number {
return requireConfig().embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS;
}
/**
* #2552: cap for parallel bulk-embed workers against a local inference
* server. A single-slot Ollama/llama-server serializes requests, so the
* cloud-tuned 20-worker fan-out multiplies latency x20 and blows past the
* fetch timeout with no surfaced error (the backfill silently starves).
*/
export const LOCAL_EMBED_CONCURRENCY_CAP = 2;
/**
* #2552: true when the configured embedding model routes to a local
* inference server — the `ollama` / `llama-server` recipes, or any recipe
* whose base URL was explicitly pointed at localhost. Bulk callers use this
* to pick CPU-safe concurrency defaults; `gbrain doctor` uses it to warn
* about an explicit cloud-sized override. Fail-open: unconfigured or
* unresolvable gateway → false (cloud behavior, the historical default).
*/
export function isLocalEmbeddingEndpoint(): boolean {
try {
const { recipe } = resolveRecipe(getEmbeddingModel());
if (recipe.id === 'ollama' || recipe.id === 'llama-server') return true;
const base = requireConfig().base_urls?.[recipe.id] ?? '';
return /\/\/(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(base);
} catch {
return false;
}
}
/**
* v0.28.11: returns the configured multimodal embedding model when set,
* or undefined if the brain falls back to `embedding_model` for multimodal
+3 -11
View File
@@ -29,17 +29,9 @@ export const ollama: Recipe = {
trust_custom_dims: true, // #2271: local models carry varied native dims
cost_per_1m_tokens_usd: 0,
price_last_verified: '2026-04-20',
// #2552: Ollama's true batch capacity depends on the locally loaded
// model + OLLAMA_NUM_PARALLEL, but the previous `no_batch_cap: true`
// meant a whole page went out in ONE request — on a CPU-only box that
// multiplies latency past the fetch timeout and the backfill starves
// with no surfaced error. Ollama doesn't return a recognizable
// token-limit error either, so the recursive-halving safety net never
// fires; a conservative static pre-split cap is the only guard.
// 4096 tokens x 2 chars/token ~= 8K chars per request (code-dense
// pages run ~2 chars/token, not the tiktoken-ish 4).
max_batch_tokens: 4096,
chars_per_token: 2,
// Ollama's batch capacity depends on the locally loaded model + the
// OLLAMA_NUM_PARALLEL config; no static cap to declare. v0.32 (#779).
no_batch_cap: true,
},
},
setup_hint: 'Install Ollama from https://ollama.ai, then `ollama pull nomic-embed-text` and `ollama serve`.',
+3
View File
@@ -61,7 +61,10 @@ export const MAX_OUTPUT_TOKENS_CEIL = 32_000;
* (with a readable error) instead of the provider's opaque HTTP 400.
*/
export const ANTHROPIC_OUTPUT_CAPS: Record<string, number> = {
'claude-fable-5': 64_000,
'claude-opus-4-8': 32_000,
'claude-opus-4-7': 32_000,
'claude-sonnet-5': 64_000,
'claude-sonnet-4-6': 64_000,
'claude-haiku-4-5': 64_000,
'claude-haiku-4-5-20251001': 64_000,
+9 -1
View File
@@ -32,6 +32,7 @@
*/
import type { BrainEngine } from '../engine.ts';
import { resolveModel } from '../model-config.ts';
import { chat as defaultChat, embedQuery, type ChatResult, type ChatOpts } from '../ai/gateway.ts';
import { hybridSearch, hybridSearchCached } from '../search/hybrid.ts';
import { fetchFar, type CloseRef, type FarPage } from './domain-bank.ts';
@@ -538,7 +539,14 @@ async function _runBrainstormInner(
const embedFn = opts.embedQueryFn ?? embedQuery;
// ---- Phase 0: cost preview + TTY grace ----
const modelStr = opts.modelOverride ?? 'anthropic:claude-sonnet-4-6';
// Tier-resolved (mirrors the cycle phases): honors models.brainstorm >
// models.default > models.tier.reasoning; the fallback keeps stock
// behavior identical (reasoning tier default IS claude-sonnet-4-6).
const modelStr = opts.modelOverride ?? await resolveModel(engine, {
configKey: 'models.brainstorm',
tier: 'reasoning',
fallback: 'anthropic:claude-sonnet-4-6',
});
const { aborted, estimate } = await previewCostAndWait({
profile,
model: modelStr,
+13 -3
View File
@@ -26,8 +26,8 @@
*/
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { chat as gatewayChat } from '../ai/gateway.ts';
import { TIER_DEFAULTS } from '../model-config.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { resolveModel } from '../model-config.ts';
import { gateVoice, type VoiceGateGenerator, type VoiceGateJudge } from '../calibration/voice-gate.ts';
import { patternStatementTemplate, type PatternStatementSlots } from '../calibration/templates.ts';
// v0.41 T10 — domain widening. The aggregator module resolves the active
@@ -229,7 +229,16 @@ class CalibrationProfilePhase extends BaseCyclePhase {
): Promise<{ summary: string; details: Record<string, unknown>; status?: PhaseStatus }> {
const holder = opts.holder ?? 'garry';
const promptVersion = opts.promptVersion ?? CALIBRATION_PROFILE_PROMPT_VERSION;
const modelId = opts.model ?? TIER_DEFAULTS.reasoning;
// Resolved once (see propose-takes.ts for the chain): models.calibration_profile
// > models.default > env > the gateway's chat model (itself resolved
// through models.chat + the reasoning tier). Provider-prefixed per #2451
// — a bare id would make gateway.chat() throw "missing a provider
// prefix". Drives the generator's chat call, the budget label, and the
// persisted model_id, so the three can never disagree.
const modelId = opts.model ?? await resolveModel(engine, {
configKey: 'models.calibration_profile',
fallback: getChatModel(),
});
const gradeCompletion = opts.gradeCompletion ?? 1.0;
const patternsGenerator = opts.patternsGenerator ?? defaultPatternsGenerator;
const biasTagsGenerator = opts.biasTagsGenerator ?? defaultBiasTagsGenerator;
@@ -265,6 +274,7 @@ class CalibrationProfilePhase extends BaseCyclePhase {
scorecard,
holder,
attempt,
modelHint: modelId,
...(feedback !== undefined ? { feedback } : {}),
});
return lines.join('\n');
+22 -3
View File
@@ -36,7 +36,9 @@
import { createHash } from 'node:crypto';
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { chat as gatewayChat } from '../ai/gateway.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { resolveModel } from '../model-config.ts';
import { splitProviderModelId } from '../model-id.ts';
import { GBrainError } from '../types.ts';
import type { OperationContext } from '../operations.ts';
import type { BrainEngine, Take, TakeResolution } from '../engine.ts';
@@ -395,7 +397,24 @@ class GradeTakesPhase extends BaseCyclePhase {
const autoResolve = opts.autoResolve ?? false; // D17 default OFF
const autoResolveThreshold = opts.autoResolveThreshold ?? 0.95; // D12 conservative
const resolvedByLabel = opts.resolvedByLabel ?? 'gbrain:grade_takes';
const judgeModelId = opts.model ?? 'claude-sonnet-4-6';
// Resolve the judge model ONCE (see propose-takes.ts for the chain —
// same label-vs-actual split fixed here: the judge call rode the
// gateway's chat_model while the grade cache key, evidence signature,
// and budget label recorded a hardcoded 4.6).
// NOTE: changing the resolved judge model invalidates the grade cache
// (judge_model_id is part of its key) — a one-time, budget-capped
// re-grade wave that is CORRECT, since the actual judge did change.
const judgeModelFull = opts.model ?? await resolveModel(engine, {
configKey: 'models.grade_takes',
fallback: getChatModel(),
});
// Bare tail for cache keys / evidence signatures / stored ids — the
// grade cache has always been keyed on bare ids; the gateway default
// resolves provider-prefixed, and normalizing preserves cache continuity
// on stock installs (no spurious re-judge wave from a prefix change). A
// genuinely different configured judge still invalidates, which is
// correct. The FULL string drives the actual judge call.
const judgeModelId = splitProviderModelId(judgeModelFull).model || judgeModelFull;
const useEnsemble = opts.useEnsemble ?? false;
const ensembleThreshold = opts.ensembleThreshold ?? 0.85;
@@ -468,7 +487,7 @@ class GradeTakesPhase extends BaseCyclePhase {
// Call the single-model judge. Errors on a single take log warning + continue.
let verdict: JudgeVerdict;
try {
verdict = await judge({ take, evidence, modelHint: opts.model });
verdict = await judge({ take, evidence, modelHint: judgeModelFull });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
result.warnings.push(`judge failed on take ${take.id}: ${msg}`);
+58 -16
View File
@@ -5,11 +5,16 @@
* a tuned LLM extractor, writes the extracted gradeable claims to the
* `take_proposals` queue. User accepts/rejects via `gbrain takes propose`.
*
* Idempotency contract (D17 schema spec):
* The unique index on (source_id, page_slug, content_hash, prompt_version)
* means an unchanged page never re-spends LLM tokens. Bumping
* PROPOSE_TAKES_PROMPT_VERSION cleanly invalidates the cache so a tuned
* prompt re-runs proposals on every page.
* Idempotency contract (D17 schema spec; per-claim rows since migration v125):
* Every scan of a (source_id, page_slug, content_hash, prompt_version)
* tuple leaves at least one row — one per extracted claim, or a single
* status='empty' sentinel when extraction yields nothing — so an unchanged
* page never re-spends LLM tokens. Pre-v125 only proposal rows were
* written: a zero-claim page never entered the cache and was re-extracted
* on EVERY cycle (observed live: ~60 such pages × every cycle ≈ 1,400
* wasted extractor calls / ~$15 per day — ~90% of total autopilot spend).
* Bumping PROPOSE_TAKES_PROMPT_VERSION cleanly invalidates the cache so a
* tuned prompt re-runs proposals on every page.
*
* F2 fence dedup:
* The phase reads the page's existing `<!-- gbrain:takes:begin -->` fence
@@ -40,6 +45,7 @@
import { randomUUID, createHash } from 'node:crypto';
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { resolveModel } from '../model-config.ts';
import { writeReceipt } from '../extract/receipt-writer.ts';
import { upsertExtractRollup } from '../extract/rollup-writer.ts';
import { GBrainError } from '../types.ts';
@@ -307,6 +313,18 @@ class ProposeTakesPhase extends BaseCyclePhase {
const promptVersion = opts.promptVersion ?? PROPOSE_TAKES_PROMPT_VERSION;
const pageLimit = opts.pageLimit ?? 100;
const skipPagesWithFence = opts.skipPagesWithFence ?? false;
// Resolve the extractor model ONCE: models.propose_takes >
// models.default > GBRAIN_MODEL env > the gateway's chat model (which
// reconfigureGatewayWithEngine already resolved through models.chat +
// the reasoning tier). One resolved provider-prefixed string drives the
// actual chat call, the budget estimate, AND the stored model_id — so
// the recorded model can never disagree with the model that ran (#2451
// convention: stored ids are provider-prefixed, nested prefixes like
// openrouter:anthropic/... stay intact).
const extractorModelId = opts.model ?? await resolveModel(engine, {
configKey: 'models.propose_takes',
fallback: getChatModel(),
});
const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`;
const result: ProposeTakesResult = {
@@ -330,8 +348,6 @@ class ProposeTakesPhase extends BaseCyclePhase {
opts.reporter.start('propose_takes.pages' as never, pages.length);
}
const modelId = opts.model ?? getChatModel();
for (const page of pages) {
result.pages_scanned += 1;
this.tick(opts);
@@ -361,7 +377,7 @@ class ProposeTakesPhase extends BaseCyclePhase {
// Budget pre-check before the LLM call. Estimate: ~1500 input tokens + 500 output.
const budget = this.checkBudget({
modelId,
modelId: extractorModelId,
estimatedInputTokens: 1500,
maxOutputTokens: 500,
});
@@ -380,7 +396,7 @@ class ProposeTakesPhase extends BaseCyclePhase {
pagePath: page.slug,
pageBody: body,
existingTakes,
modelHint: opts.model,
modelHint: extractorModelId,
});
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
@@ -388,16 +404,42 @@ class ProposeTakesPhase extends BaseCyclePhase {
continue;
}
// Write proposals to take_proposals. Each row is a separate INSERT
// because the composite idempotency key is on the per-page tuple — a
// bulk UPSERT would collapse a same-page-multi-claim run into one row.
for (const p of proposals) {
// Zero-claim scans MUST still enter the idempotency cache. Pre-v125
// only proposal rows were written, so a page whose extraction yielded
// no gradeable claims never got a row for its (page, content_hash) —
// the cache check above missed on every subsequent cycle and the LLM
// call was re-spent on the same unchanged page, forever. The sentinel
// row (status='empty', empty claim_text) is invisible to the review
// queue (pending_idx is partial on status='pending'); it exists only
// so the cache check hits.
if (proposals.length === 0) {
await engine.executeRaw(
`INSERT INTO take_proposals
(source_id, page_slug, content_hash, prompt_version, proposal_run_id,
status, claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id)
VALUES ($1, $2, $3, $4, $5, 'empty', '', 'none', 'brain', 0, NULL, NULL, $6)
ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING`,
[sourceId, page.slug, ch, promptVersion, proposalRunId, extractorModelId],
);
continue;
}
// Write proposals to take_proposals, one row per claim. The v125
// idempotency index includes md5(claim_text), so a same-page
// multi-claim run keeps EVERY claim — the pre-v125 four-column unique
// index made claims 2..N conflict with claim 1 and ON CONFLICT DO
// NOTHING silently dropped them (the review queue only ever saw the
// first claim of each page version). RETURNING id keeps
// proposals_inserted honest: it counts rows that actually landed,
// not insert attempts.
for (const p of proposals) {
const landed = await engine.executeRaw<{ id: number }>(
`INSERT INTO take_proposals
(source_id, page_slug, content_hash, prompt_version, proposal_run_id,
claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (source_id, page_slug, content_hash, prompt_version) DO NOTHING`,
ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING
RETURNING id`,
[
sourceId,
page.slug,
@@ -410,10 +452,10 @@ class ProposeTakesPhase extends BaseCyclePhase {
p.weight,
p.domain ?? null,
JSON.stringify(existingTakes),
modelId,
extractorModelId,
],
);
result.proposals_inserted += 1;
if (landed.length > 0) result.proposals_inserted += 1;
}
}
+3
View File
@@ -58,8 +58,11 @@ const SUMMARY_SLUG_RE = /^[a-z0-9][a-z0-9\-]*(\/[a-z0-9][a-z0-9\-]*)*$/;
* resolver returns for known Anthropic aliases.
*/
const MODEL_CONTEXT_TOKENS: Record<string, number> = {
'claude-fable-5': 1_000_000,
'claude-opus-4-8': 1_000_000,
'claude-opus-4-7': 1_000_000,
'claude-opus-4-6': 1_000_000,
'claude-sonnet-5': 1_000_000,
'claude-sonnet-4-6': 200_000,
'claude-sonnet-4-5': 200_000,
'claude-haiku-4-5-20251001': 200_000,
-1
View File
@@ -141,7 +141,6 @@ export const OPS_CHECK_NAMES: ReadonlySet<string> = new Set([
'pgbouncer_prepare',
'pgvector',
'pool_budget',
'embed_concurrency',
'progressive_batch_audit_health',
'queue_health',
'reranker_health',
+35
View File
@@ -5671,6 +5671,41 @@ export const MIGRATIONS: Migration[] = [
`);
},
},
{
version: 125,
name: 'take_proposals_empty_scan_sentinels_and_per_claim_rows',
// v0.42.x — kill the propose_takes rescan loop + stop dropping claims.
//
// Two defects, one schema touch:
// 1. Zero-claim scans never entered the idempotency cache (only
// proposal rows were written), so pages whose extraction yielded
// nothing were re-extracted on EVERY cycle. Observed live: ~60
// such pages per run ≈ 1,400 wasted extractor calls / ~$15 per
// day — ~90% of total autopilot LLM spend. Fix: the phase now
// writes a status='empty' sentinel row per zero-claim scan; the
// status CHECK gains the 'empty' value. Sentinels are excluded
// from the partial pending index, so the review queue never sees
// them.
// 2. The 4-column unique index collapsed a same-page multi-claim run
// to its FIRST claim: rows 2..N conflicted and ON CONFLICT DO
// NOTHING silently dropped them (verified live: exactly one row
// per (page, hash) across 3 days of runs). Fix: the idempotency
// index gains md5(claim_text) — per-claim rows, while the
// 4-column prefix still serves the per-scan cache lookup.
//
// Existing data is index-safe by construction: the old index guaranteed
// at most one row per 4-tuple, so the widened index has no duplicates
// to trip on. The DROP+ADD CONSTRAINT pair is idempotent as a unit.
idempotent: true,
sql: `
ALTER TABLE take_proposals DROP CONSTRAINT IF EXISTS take_proposals_status_check;
ALTER TABLE take_proposals ADD CONSTRAINT take_proposals_status_check
CHECK (status IN ('pending','accepted','rejected','superseded','empty'));
DROP INDEX IF EXISTS take_proposals_idempotency_idx;
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
`,
},
];
export const LATEST_VERSION = MIGRATIONS.length > 0
+7 -16
View File
@@ -56,7 +56,7 @@ import { GBrainError, PAGE_SORT_SQL, ENRICH_ORDER_SQL } from './types.ts';
import { finalizeLastSeen } from './chronicle/last-seen.ts';
import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts';
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery, buildWebsearchQueryExpr } from './search/sql-ranking.ts';
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
import {
normalizeEngineColumn,
buildVectorCastFragment,
@@ -1591,8 +1591,6 @@ export class PGLiteEngine implements BrainEngine {
}
// v0.20.0 Cathedral II Layer 10 C1/C2: language + symbol-kind filters.
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query, innerLimit, limit, offset];
let extraFilter = '';
if (opts?.language) {
@@ -1632,7 +1630,6 @@ export class PGLiteEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
const keywordSql =
`WITH ranked AS (
@@ -1640,14 +1637,14 @@ export class PGLiteEngine implements BrainEngine {
p.slug, p.id as page_id, p.title, p.type, p.source_id,
p.effective_date, p.effective_date_source,
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
ts_rank(cc.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score,
ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score,
CASE WHEN p.updated_at < (
SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id
) THEN true ELSE false END AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
JOIN sources s ON s.id = p.source_id
WHERE cc.search_vector @@ ${ftsQueryExpr} ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause}
WHERE cc.search_vector @@ websearch_to_tsquery('${ftsLang}', $1) ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause}
-- v0.27.1: hide image rows from default text-keyword search so
-- OCR text doesn't drown text-page hits. Image-similarity queries
-- run a separate vector path on embedding_image.
@@ -1715,10 +1712,7 @@ export class PGLiteEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query, limit, offset];
let extraFilter = '';
if (opts?.type) {
@@ -1766,7 +1760,7 @@ export class PGLiteEngine implements BrainEngine {
COALESCE(rep.chunk_index, 0) as chunk_index,
COALESCE(rep.chunk_text, '') as chunk_text,
COALESCE(rep.chunk_source, 'compiled_truth') as chunk_source,
ts_rank_cd(p.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score,
ts_rank_cd(p.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score,
CASE WHEN p.updated_at < (
SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id
) THEN true ELSE false END AS stale
@@ -1781,7 +1775,7 @@ export class PGLiteEngine implements BrainEngine {
ORDER BY (cc.chunk_source = 'compiled_truth') DESC, cc.chunk_index ASC
LIMIT 1
) rep ON true
WHERE p.search_vector @@ ${ftsQueryExpr}
WHERE p.search_vector @@ websearch_to_tsquery('${ftsLang}', $1)
${extraFilter} ${hardExcludeClause} ${visibilityClause}
ORDER BY score DESC, p.id ASC
LIMIT $2 OFFSET $3`;
@@ -1968,8 +1962,6 @@ export class PGLiteEngine implements BrainEngine {
});
}
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query, limit, offset];
let extraFilter = '';
if (opts?.language) {
@@ -2004,21 +1996,20 @@ export class PGLiteEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
const { rows } = await this.db.query(
`SELECT
p.slug, p.id as page_id, p.title, p.type, p.source_id,
p.effective_date, p.effective_date_source,
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
ts_rank(cc.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score,
ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score,
CASE WHEN p.updated_at < (
SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id
) THEN true ELSE false END AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
JOIN sources s ON s.id = p.source_id
WHERE cc.search_vector @@ ${ftsQueryExpr} ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause}
WHERE cc.search_vector @@ websearch_to_tsquery('${ftsLang}', $1) ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause}
ORDER BY score DESC
LIMIT $2 OFFSET $3`,
params
+2 -2
View File
@@ -762,7 +762,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
proposed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
proposal_run_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending','accepted','rejected','superseded')),
CHECK (status IN ('pending','accepted','rejected','superseded','empty')),
claim_text TEXT NOT NULL,
kind TEXT NOT NULL,
holder TEXT NOT NULL,
@@ -777,7 +777,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
+7 -16
View File
@@ -64,7 +64,7 @@ import { ConnectionManager } from './connection-manager.ts';
import { logConnectionEvent } from './connection-audit.ts';
import { validateSlug, contentHash, rowToPage, rowToStalePage, rowToChunk, rowToSearchResult, parseEmbedding, tryParseEmbedding, takeRowToTake, takeHitRowToHit, isUndefinedTableError, warnOncePerProcess } from './utils.ts';
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery, buildWebsearchQueryExpr } from './search/sql-ranking.ts';
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts';
import { DELETE_BATCH_SIZE } from './engine-constants.ts';
@@ -1691,8 +1691,6 @@ export class PostgresEngine implements BrainEngine {
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query];
let typeClause = '';
if (type) {
@@ -1763,7 +1761,6 @@ export class PostgresEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
const rawQuery = `
WITH ranked_chunks AS (
@@ -1771,11 +1768,11 @@ export class PostgresEngine implements BrainEngine {
p.slug, p.id as page_id, p.title, p.type, p.source_id,
p.effective_date, p.effective_date_source,
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
ts_rank(cc.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score
ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
JOIN sources s ON s.id = p.source_id
WHERE cc.search_vector @@ ${ftsQueryExpr}
WHERE cc.search_vector @@ websearch_to_tsquery('${ftsLang}', $1)
${typeClause}
${typesClause}
${excludeSlugsClause}
@@ -1866,10 +1863,7 @@ export class PostgresEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query];
let typeClause = '';
if (opts?.type) {
@@ -1929,7 +1923,7 @@ export class PostgresEngine implements BrainEngine {
COALESCE(rep.chunk_index, 0) as chunk_index,
COALESCE(rep.chunk_text, '') as chunk_text,
COALESCE(rep.chunk_source, 'compiled_truth') as chunk_source,
ts_rank_cd(p.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score,
ts_rank_cd(p.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score,
false AS stale
FROM pages p
JOIN sources s ON s.id = p.source_id
@@ -1942,7 +1936,7 @@ export class PostgresEngine implements BrainEngine {
ORDER BY (cc.chunk_source = 'compiled_truth') DESC, cc.chunk_index ASC
LIMIT 1
) rep ON true
WHERE p.search_vector @@ ${ftsQueryExpr}
WHERE p.search_vector @@ websearch_to_tsquery('${ftsLang}', $1)
${typeClause}
${typesClause}
${excludeSlugsClause}
@@ -2006,8 +2000,6 @@ export class PostgresEngine implements BrainEngine {
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
// #2380: slash-bearing queries match both the split-word and literal
// slash forms — see buildWebsearchQueryExpr in ./search/sql-ranking.ts.
const params: unknown[] = [query];
let typeClause = '';
if (type) {
@@ -2068,19 +2060,18 @@ export class PostgresEngine implements BrainEngine {
// FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
// — safe to interpolate into raw SQL.
const ftsLang = getFtsLanguage();
const ftsQueryExpr = buildWebsearchQueryExpr(ftsLang, '$1', query);
const rawQuery = `
SELECT
p.slug, p.id as page_id, p.title, p.type, p.source_id,
p.effective_date, p.effective_date_source,
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
ts_rank(cc.search_vector, ${ftsQueryExpr}) * ${sourceFactorCase} AS score,
ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score,
false AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
JOIN sources s ON s.id = p.source_id
WHERE cc.search_vector @@ ${ftsQueryExpr}
WHERE cc.search_vector @@ websearch_to_tsquery('${ftsLang}', $1)
${typeClause}
${typesClause}
${excludeSlugsClause}
+10 -4
View File
@@ -1274,8 +1274,14 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx
WHERE published = true;
-- take_proposals: propose_takes phase queue. Idempotency cache via the
-- composite unique index (source_id, page_slug, content_hash, prompt_version)
-- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run.
-- composite unique index (source_id, page_slug, content_hash, prompt_version,
-- md5(claim_text)) — the 4-column prefix is the per-scan cache key (mirrors
-- v0.23 dream_verdicts); md5(claim_text) makes rows per-claim so multi-claim
-- pages keep every claim (v125). status='empty' rows are zero-claim scan
-- sentinels: they hold the cache slot for a page version whose extraction
-- yielded nothing — the phase never re-spends the LLM call on that page
-- version. Excluded from the partial pending index. proposal_run_id supports
-- --rollback by run.
CREATE TABLE IF NOT EXISTS take_proposals (
id BIGSERIAL PRIMARY KEY,
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
@@ -1286,7 +1292,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
proposed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
proposal_run_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending','accepted','rejected','superseded')),
CHECK (status IN ('pending','accepted','rejected','superseded','empty')),
claim_text TEXT NOT NULL,
kind TEXT NOT NULL,
holder TEXT NOT NULL,
@@ -1301,7 +1307,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
-22
View File
@@ -251,28 +251,6 @@ export function buildOrFallbackWebsearchQuery(query: string): string | null {
return tokens.join(' OR ');
}
/**
* #2380: FTS query expression for slash-bearing queries. Postgres' default
* text-search parser classifies `foo/bar` as a single `file`-alias lexeme —
* on BOTH the query side and the index side. So a raw `foo/bar` query only
* matched documents carrying the identical joined lexeme (literal paths),
* and a slash-split query only matches documents whose text had the words
* separated. Neither form alone covers both document shapes; OR the two
* parses so a slash query matches prose ("foo and bar", stemmed, AND
* semantics) AND literal slash forms ("src/core/x.ts") alike.
*
* Slash-free queries return the plain single-parse expression — byte-
* identical SQL and identical ts_rank to the historical behavior.
*
* `ftsLang` is validated by getFtsLanguage() (safe to interpolate);
* `param` is a `$N` placeholder, never user text.
*/
export function buildWebsearchQueryExpr(ftsLang: string, param: string, query: string): string {
const plain = `websearch_to_tsquery('${ftsLang}', ${param})`;
if (!query.includes('/')) return plain;
return `(websearch_to_tsquery('${ftsLang}', translate(${param}, '/', ' ')) || ${plain})`;
}
// ============================================================
// v0.29.1 — Recency component SQL builder
// ============================================================
+10 -4
View File
@@ -1270,8 +1270,14 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx
WHERE published = true;
-- take_proposals: propose_takes phase queue. Idempotency cache via the
-- composite unique index (source_id, page_slug, content_hash, prompt_version)
-- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run.
-- composite unique index (source_id, page_slug, content_hash, prompt_version,
-- md5(claim_text)) — the 4-column prefix is the per-scan cache key (mirrors
-- v0.23 dream_verdicts); md5(claim_text) makes rows per-claim so multi-claim
-- pages keep every claim (v125). status='empty' rows are zero-claim scan
-- sentinels: they hold the cache slot for a page version whose extraction
-- yielded nothing — the phase never re-spends the LLM call on that page
-- version. Excluded from the partial pending index. proposal_run_id supports
-- --rollback by run.
CREATE TABLE IF NOT EXISTS take_proposals (
id BIGSERIAL PRIMARY KEY,
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
@@ -1282,7 +1288,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
proposed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
proposal_run_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending','accepted','rejected','superseded')),
CHECK (status IN ('pending','accepted','rejected','superseded','empty')),
claim_text TEXT NOT NULL,
kind TEXT NOT NULL,
holder TEXT NOT NULL,
@@ -1297,7 +1303,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
@@ -28,8 +28,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
resetGateway();
});
test('LiteLLM and llama-server declare no_batch_cap: true', () => {
for (const id of ['litellm', 'llama-server']) {
test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => {
for (const id of ['ollama', 'litellm', 'llama-server']) {
const r = getRecipe(id);
expect(r, `${id} not registered`).toBeDefined();
expect(
@@ -39,18 +39,6 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
}
});
test('#2552: Ollama declares a conservative static batch cap, not no_batch_cap', () => {
// A CPU-only Ollama box wedges when a whole page ships in one request;
// Ollama never returns a token-limit error so the recursive-halving
// safety net can't fire. The pre-split cap is the only guard.
const r = getRecipe('ollama');
expect(r).toBeDefined();
const e = r!.touchpoints.embedding!;
expect(e.no_batch_cap).toBeUndefined();
expect(e.max_batch_tokens).toBe(4096);
expect(e.chars_per_token).toBe(2);
});
test('configureGateway does NOT warn for ollama/litellm/llama-server', () => {
warnSpy.mockClear();
resetGateway();
+1
View File
@@ -38,6 +38,7 @@ function buildMockEngine(opts: { scorecard: TakesScorecard }): {
} {
const captured: CapturedSql[] = [];
const engine = {
async getConfig() { return null; },
kind: 'pglite',
async getScorecard() {
return opts.scorecard;
-118
View File
@@ -1,118 +0,0 @@
/**
* #2552: cloud-tuned embedding defaults silently wedge CPU-only local
* endpoints (Ollama). Three-part fix under test:
*
* 1. `isLocalEmbeddingEndpoint()` — gateway helper detecting local
* inference servers (ollama / llama-server recipes, localhost base URL).
* 2. `resolveEmbedConcurrency()` — embed auto-caps the 20-worker fan-out
* at LOCAL_EMBED_CONCURRENCY_CAP for local endpoints unless the
* operator set GBRAIN_EMBED_CONCURRENCY explicitly.
* 3. `computeEmbedConcurrencyCheck()` — doctor warns when an explicit env
* override fans out against a local endpoint.
*
* Serial: mutates process.env and the module-global gateway config.
*/
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
import {
configureGateway,
resetGateway,
isLocalEmbeddingEndpoint,
LOCAL_EMBED_CONCURRENCY_CAP,
} from '../src/core/ai/gateway.ts';
import { resolveEmbedConcurrency } from '../src/commands/embed.ts';
import { computeEmbedConcurrencyCheck } from '../src/commands/doctor.ts';
const SAVED_ENV = process.env.GBRAIN_EMBED_CONCURRENCY;
afterEach(() => {
resetGateway();
if (SAVED_ENV === undefined) delete process.env.GBRAIN_EMBED_CONCURRENCY;
else process.env.GBRAIN_EMBED_CONCURRENCY = SAVED_ENV;
});
afterAll(() => {
resetGateway();
});
describe('#2552 isLocalEmbeddingEndpoint', () => {
test('false when the gateway is not configured (fail-open to cloud behavior)', () => {
resetGateway();
expect(isLocalEmbeddingEndpoint()).toBe(false);
});
test('true for the ollama recipe', () => {
configureGateway({ embedding_model: 'ollama:nomic-embed-text', env: {} });
expect(isLocalEmbeddingEndpoint()).toBe(true);
});
test('true for the llama-server recipe', () => {
configureGateway({ embedding_model: 'llama-server:my-gguf', env: {} });
expect(isLocalEmbeddingEndpoint()).toBe(true);
});
test('false for a cloud recipe', () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-small',
env: { OPENAI_API_KEY: 'fake' },
});
expect(isLocalEmbeddingEndpoint()).toBe(false);
});
test('true when a cloud recipe base URL is explicitly pointed at localhost', () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-small',
env: { OPENAI_API_KEY: 'fake' },
base_urls: { openai: 'http://localhost:8080/v1' },
});
expect(isLocalEmbeddingEndpoint()).toBe(true);
});
});
describe('#2552 resolveEmbedConcurrency', () => {
test('caps at LOCAL_EMBED_CONCURRENCY_CAP for a local endpoint when env is unset', () => {
delete process.env.GBRAIN_EMBED_CONCURRENCY;
configureGateway({ embedding_model: 'ollama:nomic-embed-text', env: {} });
expect(resolveEmbedConcurrency()).toBe(LOCAL_EMBED_CONCURRENCY_CAP);
});
test('explicit env override always wins, even against a local endpoint', () => {
process.env.GBRAIN_EMBED_CONCURRENCY = '10';
configureGateway({ embedding_model: 'ollama:nomic-embed-text', env: {} });
expect(resolveEmbedConcurrency()).toBe(10);
});
test('cloud endpoints keep the historical default of 20', () => {
delete process.env.GBRAIN_EMBED_CONCURRENCY;
configureGateway({ env: { OPENAI_API_KEY: 'fake' } });
expect(resolveEmbedConcurrency()).toBe(20);
});
test('pacing only ever lowers concurrency', () => {
delete process.env.GBRAIN_EMBED_CONCURRENCY;
configureGateway({ embedding_model: 'ollama:nomic-embed-text', env: {} });
expect(resolveEmbedConcurrency(1)).toBe(1);
expect(resolveEmbedConcurrency(16)).toBe(LOCAL_EMBED_CONCURRENCY_CAP);
});
});
describe('#2552 computeEmbedConcurrencyCheck (doctor)', () => {
test('ok for non-local endpoints', () => {
expect(computeEmbedConcurrencyCheck(false, '20', 2).status).toBe('ok');
});
test('warn when an explicit override exceeds the local cap', () => {
const check = computeEmbedConcurrencyCheck(true, '20', 2);
expect(check.status).toBe('warn');
expect(check.message).toContain('GBRAIN_EMBED_CONCURRENCY=20');
});
test('ok when env is unset against a local endpoint (auto-cap applies)', () => {
expect(computeEmbedConcurrencyCheck(true, undefined, 2).status).toBe('ok');
});
test('ok when the override is at or under the cap', () => {
expect(computeEmbedConcurrencyCheck(true, '2', 2).status).toBe('ok');
expect(computeEmbedConcurrencyCheck(true, '1', 2).status).toBe('ok');
});
});
+1
View File
@@ -47,6 +47,7 @@ function buildMockEngine(opts: { takes: Take[] }): {
const captured: CapturedSql[] = [];
const resolves: CapturedResolve[] = [];
const engine = {
async getConfig() { return null; },
kind: 'pglite',
async listTakes() {
return opts.takes;
+24
View File
@@ -46,12 +46,14 @@ interface CapturedResolve {
function buildMockEngine(opts: {
takes: Take[];
cachedGrades?: Set<string>; // composite-key strings already in take_grade_cache
config?: Record<string, string>; // engine.getConfig plane (models.grade_takes etc.)
}): { engine: BrainEngine; captured: CapturedSql[]; resolves: CapturedResolve[] } {
const captured: CapturedSql[] = [];
const resolves: CapturedResolve[] = [];
const cached = opts.cachedGrades ?? new Set<string>();
const engine = {
async getConfig(key: string) { return opts.config?.[key] ?? null; },
kind: 'pglite',
async listTakes() {
return opts.takes;
@@ -224,6 +226,28 @@ describe('runPhaseGradeTakes — phase integration', () => {
expect(resolves).toHaveLength(0); // no canonical mutation
});
test('models.grade_takes config drives the judge call; cache key stays bare-tailed', async () => {
// Pre-fix the judge call rode the gateway's chat_model while the cache
// key / budget label recorded a hardcoded 'claude-sonnet-4-6'. The phase
// now resolves models.grade_takes; the judge gets the FULL string and
// the cache row keys on the bare tail (continuity with historical rows).
const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })];
const { engine, captured } = buildMockEngine({
takes,
config: { 'models.grade_takes': 'anthropic:claude-sonnet-5' },
});
const hints: Array<string | undefined> = [];
const judge: JudgeFn = async ({ modelHint }) => {
hints.push(modelHint);
return { verdict: 'correct', confidence: 0.9, reasoning: 'held' };
};
const result = await runPhaseGradeTakes(buildCtx(engine), { judge });
expect(result.status).toBe('ok');
expect(hints).toEqual(['anthropic:claude-sonnet-5']); // actual call gets the FULL string
const inserts = captured.filter(c => c.sql.includes('INSERT INTO take_grade_cache'));
expect(inserts[0]!.params[2]).toBe('claude-sonnet-5'); // judge_model_id is the bare tail
});
test('D17: auto-resolve OFF by default — even high-confidence verdict does NOT mutate takes', async () => {
const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })];
const { engine, resolves } = buildMockEngine({ takes });
-61
View File
@@ -216,67 +216,6 @@ describe('PGLiteEngine: Search', () => {
expect(results.length).toBe(0);
});
// Regression (#2380): queries containing `/` used to bypass FTS AND
// semantics. Postgres' default text-search parser classifies `foo/bar` as
// a `file`-alias token mapped to the `simple` dictionary, so it became a
// single un-stemmed lexeme `'foo/bar'` that never matches indexed text —
// the primary FTS pass returned 0 and the OR fallback took over, matching
// pages that contain EITHER term. searchKeyword/searchTitles now normalize
// `/` to whitespace before websearch_to_tsquery parses, so the primary
// AND pass matches directly.
test('searchKeyword: slash query matches with AND semantics, not OR fallback', async () => {
// Decoy shares only ONE of the two query terms ('enterprise').
await engine.putPage('concepts/enterprise-pricing', {
type: 'concept', title: 'Widget Pricing',
compiled_truth: 'Enterprise pricing for widgets.',
});
await engine.upsertChunks('concepts/enterprise-pricing', [
{ chunk_index: 0, chunk_text: 'Enterprise pricing for widgets', chunk_source: 'compiled_truth' },
]);
// Both terms co-occur only in the novamind chunk. Pre-fix this returned
// BOTH pages (primary pass zero-hit → OR fallback); post-fix the primary
// AND pass returns exactly the co-occurrence page.
const results = await engine.searchKeyword('NovaMind/enterprise');
expect(results.length).toBe(1);
expect(results[0].slug).toBe('companies/novamind');
});
test('searchTitles: slash query matches with AND semantics, not OR fallback', async () => {
await engine.putPage('companies/novamind-enterprise', {
type: 'company', title: 'NovaMind Enterprise Platform',
compiled_truth: 'Placeholder body.',
});
await engine.putPage('guides/enterprise-sales', {
type: 'concept', title: 'Enterprise Sales Guide',
compiled_truth: 'Placeholder body.',
});
// Pre-fix: `NovaMind/Enterprise` parsed as one file-alias lexeme → the
// primary title pass returned 0 and the OR fallback matched BOTH titles.
const results = await engine.searchTitles('NovaMind/Enterprise');
expect(results.length).toBe(1);
expect(results[0].slug).toBe('companies/novamind-enterprise');
});
test('searchKeyword: slash query still matches the literal slash form (file paths)', async () => {
// The INDEX side also emits the joined file-alias lexeme for literal
// `foo/bar` text, so a query normalized to split words alone would go
// blind to documents containing the literal slash form (paths, URLs).
// buildWebsearchQueryExpr ORs both parses; this pins the raw arm.
await engine.putPage('runbooks/widget-deploy', {
type: 'concept', title: 'Widget Deploy Runbook',
compiled_truth: 'Runbook for the acme/widget deployment pipeline.',
});
await engine.upsertChunks('runbooks/widget-deploy', [
{ chunk_index: 0, chunk_text: 'Runbook for the acme/widget deployment pipeline', chunk_source: 'compiled_truth' },
]);
const results = await engine.searchKeyword('acme/widget');
expect(results.length).toBe(1);
expect(results[0].slug).toBe('runbooks/widget-deploy');
});
test('tsvector trigger populates search_vector on insert', async () => {
// Verify the PL/pgSQL trigger fires and content_chunks.search_vector is
// populated from chunk_text. v0.20.0 Cathedral II Layer 3 moved FTS from
+160
View File
@@ -0,0 +1,160 @@
/**
* propose_takes rescan-loop + dropped-claims regression tests (migration v125).
*
* Two live-observed defects, both fixed by the v125 schema + phase change:
*
* 1. RESCAN LOOP a page whose extraction yielded zero claims never
* entered the idempotency cache (only proposal rows were written), so
* every cycle re-spent the extractor call on the same unchanged page.
* Live impact: ~60 such pages × every cycle 1,400 wasted LLM calls
* (~$15) per day ~90% of total autopilot spend. Fix: status='empty'
* sentinel row per zero-claim scan.
*
* 2. DROPPED CLAIMS the 4-column unique index collapsed a same-page
* multi-claim run to its first claim (rows 2..N conflicted, ON CONFLICT
* DO NOTHING dropped them silently; verified live: exactly 1 row per
* (page, hash) over 3 days). Fix: idempotency index gains
* md5(claim_text).
*
* Hermetic: PGLite engine + injected extractor; no gateway, no LLM.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { runPhaseProposeTakes, type ProposeTakesExtractor, type ProposedTake } from '../src/core/cycle/propose-takes.ts';
import type { OperationContext } from '../src/core/operations.ts';
let engine: PGLiteEngine;
function ctx(): OperationContext {
return {
engine,
remote: false,
config: {} as OperationContext['config'],
logger: { info() {}, warn() {}, error() {}, debug() {} } as unknown as OperationContext['logger'],
} as unknown as OperationContext;
}
/** Extractor stub that counts invocations per page slug. */
function countingExtractor(
claimsBySlug: Record<string, ProposedTake[]>,
): { extractor: ProposeTakesExtractor; calls: string[] } {
const calls: string[] = [];
const extractor: ProposeTakesExtractor = async ({ pagePath }) => {
calls.push(pagePath);
return claimsBySlug[pagePath] ?? [];
};
return { extractor, calls };
}
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
await engine.putPage('notes/zero-claims', {
type: 'note',
title: 'pure narrative',
compiled_truth: 'A quiet walk in the park. Nothing opinionated happened at all today.',
});
await engine.putPage('notes/three-claims', {
type: 'note',
title: 'opinionated',
compiled_truth: 'I bet acme-example wins the market. widget-co will struggle. fund-a is overexposed.',
});
});
afterAll(async () => {
await engine.disconnect();
});
describe('rescan loop — zero-claim scans enter the cache', () => {
test('second run cache-hits: extractor is NOT called again on unchanged pages', async () => {
const claims = {
'notes/three-claims': [
{ claim_text: 'acme-example wins the market', kind: 'bet' as const, holder: 'brain', weight: 0.7 },
{ claim_text: 'widget-co will struggle', kind: 'take' as const, holder: 'brain', weight: 0.6 },
{ claim_text: 'fund-a is overexposed', kind: 'take' as const, holder: 'brain', weight: 0.55 },
],
};
const first = countingExtractor(claims);
const r1 = await runPhaseProposeTakes(ctx(), { extractor: first.extractor });
expect(r1.status).toBe('ok');
// Both pages extracted on the first pass.
expect(first.calls).toContain('notes/zero-claims');
expect(first.calls).toContain('notes/three-claims');
const second = countingExtractor(claims);
const r2 = await runPhaseProposeTakes(ctx(), { extractor: second.extractor });
expect(r2.status).toBe('ok');
// THE regression: pre-fix the zero-claim page missed the cache every
// run and was re-extracted here. (Run 1's receipt page legitimately
// appears once — it's a new page — and its zero-claim scan now caches
// too; pre-fix, receipts re-scanned forever as well.)
expect(second.calls).not.toContain('notes/zero-claims');
expect(second.calls).not.toContain('notes/three-claims');
// Run 2 inserted nothing, so no new receipt page exists: run 3 must be
// fully quiescent — zero extractor calls, zero cache misses.
const third = countingExtractor(claims);
const r3 = await runPhaseProposeTakes(ctx(), { extractor: third.extractor });
expect(r3.status).toBe('ok');
expect(third.calls).toEqual([]);
expect((r3.details as Record<string, unknown>).cache_misses).toBe(0);
});
test('zero-claim scan wrote an "empty" sentinel invisible to the pending queue', async () => {
const sentinel = await engine.executeRaw<{ status: string; claim_text: string }>(
`SELECT status, claim_text FROM take_proposals WHERE page_slug = 'notes/zero-claims'`,
[],
);
expect(sentinel.length).toBe(1);
expect(sentinel[0].status).toBe('empty');
expect(sentinel[0].claim_text).toBe('');
const pending = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM take_proposals WHERE page_slug = 'notes/zero-claims' AND status = 'pending'`,
[],
);
expect(Number(pending[0].n)).toBe(0);
});
});
describe('dropped claims — per-claim rows survive the idempotency index', () => {
test('a 3-claim page stores 3 rows and reports an honest inserted count', async () => {
const rows = await engine.executeRaw<{ claim_text: string }>(
`SELECT claim_text FROM take_proposals WHERE page_slug = 'notes/three-claims' AND status = 'pending' ORDER BY id`,
[],
);
// Pre-fix the 4-column unique index kept only the FIRST claim.
expect(rows.length).toBe(3);
expect(rows.map(r => r.claim_text)).toEqual([
'acme-example wins the market',
'widget-co will struggle',
'fund-a is overexposed',
]);
});
test('content change re-extracts and stores the new version separately', async () => {
await engine.putPage('notes/zero-claims', {
type: 'note',
title: 'pure narrative',
compiled_truth: 'Updated: I now believe acme-example is undervalued and will re-rate within a year.',
});
const claims = {
'notes/zero-claims': [
{ claim_text: 'acme-example is undervalued', kind: 'take' as const, holder: 'brain', weight: 0.6 },
],
};
const run = countingExtractor(claims);
const r = await runPhaseProposeTakes(ctx(), { extractor: run.extractor });
expect(r.status).toBe('ok');
// Changed page re-extracts; the unchanged 3-claim page stays cached.
expect(run.calls).toEqual(['notes/zero-claims']);
expect((r.details as Record<string, unknown>).proposals_inserted).toBe(1);
const all = await engine.executeRaw<{ status: string }>(
`SELECT status FROM take_proposals WHERE page_slug = 'notes/zero-claims' ORDER BY id`,
[],
);
// Old hash's sentinel + new hash's pending claim coexist.
expect(all.map(r2 => r2.status).sort()).toEqual(['empty', 'pending']);
});
});
+33 -1
View File
@@ -41,12 +41,16 @@ interface CapturedSql {
function buildMockEngine(opts: {
pages: Page[];
existingProposals?: Set<string>; // composite-key strings already in take_proposals
config?: Record<string, string>; // engine.getConfig plane (models.tier.* etc.)
}): { engine: BrainEngine; captured: CapturedSql[] } {
const captured: CapturedSql[] = [];
const existing = opts.existingProposals ?? new Set<string>();
const engine = {
kind: 'pglite',
async getConfig(key: string) {
return opts.config?.[key] ?? null;
},
async listPages() {
return opts.pages;
},
@@ -59,7 +63,12 @@ function buildMockEngine(opts: {
if (existing.has(key)) return [{ id: 1 } as unknown as T];
return [];
}
// INSERT — return nothing
// INSERT ... RETURNING id — emulate a successful insert so the
// honest proposals_inserted counter (counts RETURNING rows, not
// attempts) sees the row land. Conflicted inserts would return [].
if (sql.includes('INSERT INTO take_proposals') && sql.includes('RETURNING id')) {
return [{ id: 1 } as unknown as T];
}
return [];
},
} as unknown as BrainEngine;
@@ -267,6 +276,29 @@ describe('runPhaseProposeTakes — phase integration', () => {
expect(inserts[0]!.params[9]).toBe('market'); // domain
});
test('extractor model resolves through models.propose_takes config', async () => {
// Pre-fix the phase's only model knob was the gateway chat model — there
// was no per-phase config key. The phase now resolves once via
// resolveModel(models.propose_takes > models.default > env > gateway
// chat model); the extractor hint and the stored model_id must both
// reflect the configured override (full provider-prefixed string, #2451).
const pages = [buildPage({ slug: 'wiki/concepts/tier-routing', body: 'Tier-routed models will win.' })];
const { engine, captured } = buildMockEngine({
pages,
config: { 'models.propose_takes': 'anthropic:claude-sonnet-5' },
});
const seen: Array<string | undefined> = [];
const extractor: ProposeTakesExtractor = async ({ modelHint }) => {
seen.push(modelHint);
return [{ claim_text: 'tier-routed models win', kind: 'bet', holder: 'brain', weight: 0.7 }];
};
const result = await runPhaseProposeTakes(buildCtx(engine), { extractor });
expect(result.status).toBe('ok');
expect(seen).toEqual(['anthropic:claude-sonnet-5']); // chat call gets the FULL string
const inserts = captured.filter(c => c.sql.includes('INSERT INTO take_proposals'));
expect(inserts[0]!.params[11]).toBe('anthropic:claude-sonnet-5'); // stored model_id matches the call
});
test('cache hit: page already in take_proposals is skipped', async () => {
const body = 'A page that was already processed.';
const pages = [buildPage({ slug: 'wiki/old-page', body })];