mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Merge branch 'master' into build/skills-integrity
This commit is contained in:
Time Attakc
+5
-2
@@ -1,4 +1,7 @@
|
||||
node_modules/
|
||||
# No trailing slash: a bare `node_modules/` pattern matches directories only,
|
||||
# so a *symlink* named node_modules slips past it and can be committed
|
||||
# (that's how the /tmp-pointing symlink in faf5cdba got in). Match any type.
|
||||
node_modules
|
||||
bin/
|
||||
.DS_Store
|
||||
*.log
|
||||
@@ -15,7 +18,7 @@ supabase/.temp/
|
||||
# self-contained binaries (the bun --compile path embeds it via
|
||||
# `import path from 'admin/dist/index.html' with { type: 'file' }`).
|
||||
# Build via: cd admin && bun install && bun run build.
|
||||
admin/node_modules/
|
||||
admin/node_modules
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
|
||||
@@ -189,7 +189,8 @@ per-release `**vX.Y.Z:**` narration — CI enforces this
|
||||
- `scripts/check-no-double-retry.sh` + `scripts/check-batch-audit-site.sh` — CI lint guards wired into `bun run verify`. The former greps src/ for `withRetry(...engine.{addLinksBatch|addTimelineEntriesBatch|upsertChunks})` patterns and fails the build on hit (prevents 3×3=9 retry amplification on incomplete reverts). The latter extracts every string-literal `auditSite: '...'` from src/ and validates each appears in the `BATCH_AUDIT_SITES` const in `src/core/retry.ts` (typo guard — prevents fragmented doctor output).
|
||||
- `src/core/fail-improve.ts` — Deterministic-first, LLM-fallback loop with JSONL failure logging and auto-test generation.
|
||||
- `src/core/transcription.ts` — Audio transcription: Groq Whisper (default), OpenAI fallback, ffmpeg segmentation for >25MB.
|
||||
- `src/core/enrichment-service.ts` — Global enrichment service: entity slug generation, tier auto-escalation, batch throttling.
|
||||
- `src/core/enrichment-service.ts` — Global enrichment service: entity slug generation, tier auto-escalation, batch throttling. Write path is trust-gated (issue #160): `enrichEntity` / `enrichEntities` / `extractAndEnrich` take `EnrichmentTrustOptions { trusted?, sourceId? }`; only an explicit `trusted: true` writes authoritative `people/` / `companies/` stubs. Anything else (undefined/false — fail-closed, mirroring `OperationContext.remote`) creates the stub with the extraction quarantine markers from `src/core/extraction-review.ts` and reports `quarantined: true` in `EnrichmentResult`. The ONLY sanctioned op surface is `extract_entities` (operations.ts), which grants `trusted` solely for `ctx.remote === false` callers passing `--trusted-extraction`.
|
||||
- `src/core/extraction-review.ts` — Extraction quarantine lane markers (issue #160), sibling of `src/core/quarantine.ts` / `embed-skip.ts` (frontmatter-key pattern, no schema migration). Auto-extracted stubs from untrusted input carry the PAIR `provenance: 'auto-extracted'` + `status: 'unverified'` (both required — user pages with their own `status`/`provenance` never match). Exports `quarantineMarkers()`, `isUnverifiedExtraction()` (JS predicate) and `unverifiedExtractionFragment(alias)` — the single SQL source of truth consumed by `buildSourceFactorCase` (namespace source-boost guard), both engines' `getUnverifiedExtractionPageIds`, the `extraction_pending` op, and the `unverified_extractions` doctor check, so filter and marker keys can never drift. Consequences: unverified stubs are excluded from the compiled-truth fusion boost + the `people/`/`companies/` source-boost (rank as ordinary content), stamped `unverified: true` in search results (`stampUnverifiedExtractions`, hybrid.ts), listed by `extraction_pending`, promoted (status → `verified`, provenance kept for audit) or rejected (soft-delete) by the owner-only `extraction_review` op. Pinned by `test/extraction-review.test.ts` (PGLite) + `test/e2e/extraction-review-postgres.test.ts` (live Postgres parity).
|
||||
- `src/commands/enrich.ts` + `src/core/enrich/thin.ts` + `src/core/cycle/enrich-thin.ts` — `gbrain enrich --thin`: batch-develops stub (thin) pages via **brain-internal grounded synthesis**. gbrain's model tooling sees only brain-internal context (search / get_page / facts / backlinks), not the web, so enrich consolidates what the brain ALREADY knows about an entity (scattered across meetings, other pages, deals, facts) into one cited page via ONE `gateway.chat` call per page; web research stays the agent-driven `enrich` SKILL's job. `runEnrichCore(engine, opts, signal)` (strict per-source; multi-source iteration is the caller's job) drives `enrichOne` per candidate: `withRefreshingLock('enrich:<src>:<slug>')` → `getPage` → deterministic retrieve (hybridSearch + getBacklinks + facts + raw_data, source-scoped, sanitized via `INJECTION_PATTERNS`) → `assessGrounding` gate (skip < `MIN_CONTEXT_CHARS`, no LLM) → `buildEnrichPrompt` (grounded dossier, `[Source: slug]` citations, SKIP sentinel) → synth → `put_page` handler (`remote:false`, auto-link + write-through) stamping `enriched_at` + `enriched_by:'cli:enrich'`. Candidate selection is the SQL-native `engine.listEnrichCandidates(opts)` (`src/core/engine.ts` interface + `EnrichCandidate`/`EnrichCandidatesOpts`/`ENRICH_ORDER_SQL` in `src/core/types.ts` + pg/pglite impls): thin-filter + per-page source-correct inbound count (`to_page_id = p.id`, `mentions` excluded) + `enriched_at` recency guard + whitelisted ORDER BY + LIMIT, lightweight projection (NO bodies). Resume via `src/core/op-checkpoint.ts` (local `enrichFingerprint`); budget via `BudgetTracker` + `withBudgetTracker` (best-effort under `--workers > 1` — `runSlidingPool` aborts new claims on `BUDGET_EXHAUSTED` but does NOT cancel in-flight `gateway.chat`; pin `--workers 1` for a hard ceiling). `sanitizeContext` (thin.ts) neutralizes the `<context>…</context>` data-envelope delimiters (injection escape, mirrors the `</trajectory>` convention); the `--background` multi-source fan-out idempotency key carries the run fingerprint via exported `backgroundIdempotencyKey(sid, args)` (a bare `enrich:${sid}` would return stale completed jobs); `runEnrichCore` flags `budget_exhausted` post-hoc when `tracker.totalSpent > tracker.cap` even when the gateway swallowed the final-call throw (via read-only `BudgetTracker.cap` getter); `body()` flushes the checkpoint on `BudgetExhausted` before it propagates so resume doesn't re-charge. The opt-in `enrich_thin` cycle phase (default OFF via `cycle.enrich_thin.enabled`) trickles `max_pages_per_tick` (default 3) per source with per-source cost cap enforced as `min(per_source_cap, brain_wide_remaining)` + brain-wide total + walltime caps. Wired into `cycle.ts` (`CyclePhase`/`ALL_PHASES` between `conversation_facts_backfill` and `skillopt`/`embed`; `PHASE_SCOPE='source'`; `NEEDS_LOCK`; dispatch), `cli.ts` (`CLI_ONLY` + `CLI_ONLY_SELF_HELP` + `THIN_CLIENT_REFUSED_COMMANDS` + dispatch), `jobs.ts` (Minion `enrich` handler, strict per-source, NOT in `PROTECTED_JOB_NAMES`). DI seam `opts.synthesizeFn` keeps tests hermetic (no API key, no mock.module). Pinned by `test/enrich/thin.test.ts`, `test/enrich/idempotency.test.ts`, `test/enrich-cycle-phase.test.ts`, `test/e2e/enrich-pglite.test.ts` (grew-cited, skip, ordering, multi-source, recency, resume, budget abort + checkpoint flush, final-call overage, lock-skip, provenance), `test/e2e/engine-parity.test.ts` (`listEnrichCandidates` pg↔pglite parity).
|
||||
- `src/core/data-research.ts` — Recipe validation, field extraction (MRR/ARR regex), dedup, tracker parsing, HTML stripping.
|
||||
- `src/commands/embed.ts` — `gbrain embed [--stale|--all] [--slugs ...]`. `--stale` starts with `engine.countStaleChunks()` (single SELECT count(*) WHERE embedding IS NULL, ~50 bytes wire) so a fully-embedded brain short-circuits with no further reads. When stale chunks exist, `engine.listStaleChunks()` returns just the chunks needing embeddings (slug + chunk_index + chunk_text + metadata, no `vector(1536)` payload); caller groups by slug, embeds, re-upserts via `upsertChunks`. All `console.log`/`console.error` call sites use `slog`/`serr` from `src/core/console-prefix.ts` so when `runEmbedCore` runs inside a per-source `withSourcePrefix` scope (installed by the `gbrain sync --all` worker pool) every line carries the `[<source-id>] ` prefix; standalone callers see identical output because slog/serr fall through to bare console fns outside the wrap. Every embed-write path stamps `pages.embedding_signature` via `engine.setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` so a later model/dims swap is detectable as stale. The per-slug path (`embedPage`, used by `gbrain embed <slug>` AND sync's post-import embed step) and the full-re-embed path (`embedAll`) stamp unconditionally per page. The stale path (`embedAllStale`) first calls `invalidateStaleSignatureEmbeddings` on a live run so signature-drifted pages flow through the NULL cursor, then stamps each page — but ONLY when EVERY chunk was stale this pass (a partially-stale page keeps preserved chunks of unknown provenance, so it stays unstamped rather than falsely marked current; `embed --all` fully re-embeds + stamps those). dry-run never mutates: it counts signature-drift via the widened `countStaleChunks({signature})` predicate without NULLing anything. `--include-null-signature` (#3391) lifts the NULL-signature grandfather clause: threads `includeNullSignature: true` into the invalidation + counts so pages that predate the v108 stamp re-embed too after a model swap (both engines' `countStaleChunks`/`sumStaleChunkChars`/`invalidateStaleSignatureEmbeddings` accept the flag; predicate becomes `sig IS NULL OR sig <> current`). Without the flag, a live stale run that just invalidated drifted rows probes for left-behind NULL-signature chunks and emits a loud stderr warning naming the count + the fix — mixed embedding spaces in one index are never silent. Pinned by `test/embedding-migration.test.ts` + `test/e2e/migrate-embeddings-postgres.test.ts`.
|
||||
|
||||
@@ -87,6 +87,15 @@ embedding proximity. Four layers, added after the incident in
|
||||
deciding "is this page already here, safe to NOT write a duplicate?" keys off
|
||||
`create_safety`, not a raw blended score.
|
||||
|
||||
**Extraction quarantine lane (issue #160):** pages carrying the unverified
|
||||
auto-extracted markers (frontmatter `provenance: auto-extracted` +
|
||||
`status: unverified`, see `src/core/extraction-review.ts`) rank as ordinary
|
||||
content — they are skipped by the compiled-truth fusion boost and by the
|
||||
`people/`/`companies/` namespace source-boost, and every search result from
|
||||
such a page carries `unverified: true` so agents can label the provenance.
|
||||
Promote or reject them via `gbrain extraction-pending` / `gbrain
|
||||
extraction-review`.
|
||||
|
||||
The `search` MCP/CLI op is **cheap-hybrid** (vector + keyword + RRF + pool +
|
||||
title + alias, expansion off); `query` is the full-control variant. NamedThingBench
|
||||
(`gbrain eval retrieval-quality`) gates these families on every PR. Diagnose a
|
||||
|
||||
+2
-1
@@ -48,7 +48,7 @@
|
||||
"check:system-of-record": "scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "scripts/check-cli-executable.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-proposal-pii.sh && scripts/check-test-real-names.sh && scripts/check-jsonb-pattern.sh && scripts/check-source-id-projection.sh && scripts/check-source-config-leak.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh && scripts/check-skill-brain-first.sh && scripts/check-operations-filter-bypass.sh && scripts/check-gateway-routed-no-direct-anthropic.sh && scripts/check-worker-pool-atomicity.sh && scripts/check-key-files-current-state.sh && scripts/check-no-double-retry.sh && scripts/check-batch-audit-site.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-proposal-pii.sh && scripts/check-test-real-names.sh && scripts/check-jsonb-pattern.sh && scripts/check-source-id-projection.sh && scripts/check-source-config-leak.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-tracked-symlinks.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh && scripts/check-skill-brain-first.sh && scripts/check-operations-filter-bypass.sh && scripts/check-gateway-routed-no-direct-anthropic.sh && scripts/check-worker-pool-atomicity.sh && scripts/check-key-files-current-state.sh && scripts/check-no-double-retry.sh && scripts/check-batch-audit-site.sh",
|
||||
"check:gateway-routed": "scripts/check-gateway-routed-no-direct-anthropic.sh",
|
||||
"check:worker-pool-atomicity": "scripts/check-worker-pool-atomicity.sh",
|
||||
"check:doc-history": "scripts/check-key-files-current-state.sh",
|
||||
@@ -77,6 +77,7 @@
|
||||
"check:skills-manifest": "scripts/check-skills-manifest-fresh.sh",
|
||||
"check:test-names": "scripts/check-test-real-names.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:no-tracked-symlinks": "scripts/check-no-tracked-symlinks.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "scripts/check-admin-embedded.sh",
|
||||
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any symlink is tracked in git.
|
||||
#
|
||||
# A symlink committed from a build sandbox points at a path that exists on
|
||||
# exactly one machine. Everywhere else the checkout produces a dangling
|
||||
# link, and anything that opens it fails. That is not hypothetical: commit
|
||||
# faf5cdba landed `node_modules -> /tmp/fleet/repo/node_modules`, which made
|
||||
# `bun install` abort with `ENOENT: could not open the "node_modules"
|
||||
# directory` on every fresh clone, and took `gbrain upgrade`'s bun-link path
|
||||
# down with it (the auto-upgrade runs `bun install`, so the printed manual
|
||||
# fallback failed the same way).
|
||||
#
|
||||
# .gitignore alone does not prevent this. A `node_modules/` pattern with a
|
||||
# trailing slash matches directories ONLY, so a symlink of the same name is
|
||||
# never ignored. Dropping the slash closes that hole, but `git add -f` still
|
||||
# walks straight past it. This guard is the backstop.
|
||||
#
|
||||
# The repo has no legitimate tracked symlinks, so the allowlist starts
|
||||
# empty. If you ever need one, add its exact repo-relative path to ALLOWLIST
|
||||
# below and explain why — a relative link that resolves inside the repo is
|
||||
# defensible; an absolute one almost never is.
|
||||
#
|
||||
# Usage: scripts/check-no-tracked-symlinks.sh
|
||||
# Exit: 0 when clean, 1 when a tracked symlink is found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Paths permitted to be tracked symlinks. Empty by design.
|
||||
ALLOWLIST=()
|
||||
|
||||
# Git records symlinks with mode 120000. Field 4 of `ls-files -s` is the path
|
||||
# (tab-separated from the stage number), so cut on the tab to keep paths with
|
||||
# spaces intact.
|
||||
found="$(git ls-files -s | awk '$1 == "120000"' | cut -f2- || true)"
|
||||
|
||||
if [ -n "$found" ]; then
|
||||
filtered="$found"
|
||||
for f in "${ALLOWLIST[@]:-}"; do
|
||||
[ -z "$f" ] && continue
|
||||
filtered="$(echo "$filtered" | grep -vxF "$f" || true)"
|
||||
done
|
||||
|
||||
if [ -n "$filtered" ]; then
|
||||
echo "ERROR: symlink(s) tracked in git:"
|
||||
echo
|
||||
while IFS= read -r path; do
|
||||
[ -z "$path" ] && continue
|
||||
target="$(git cat-file blob ":$path" 2>/dev/null || echo '<unreadable>')"
|
||||
echo " $path -> $target"
|
||||
done <<< "$filtered"
|
||||
echo
|
||||
echo "A committed symlink resolves on the machine that created it and"
|
||||
echo "nowhere else. Untrack it:"
|
||||
echo
|
||||
echo " git rm --cached <path>"
|
||||
echo
|
||||
echo "If the path is build output (node_modules, dist, bin), also confirm"
|
||||
echo "it is covered by .gitignore WITHOUT a trailing slash — a trailing"
|
||||
echo "slash matches directories only and lets the symlink through."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "check-no-tracked-symlinks: OK (no tracked symlinks)"
|
||||
@@ -42,6 +42,7 @@ CHECKS=(
|
||||
"check:source-id-projection"
|
||||
"check:source-config-leak"
|
||||
"check:progress"
|
||||
"check:no-tracked-symlinks"
|
||||
"check:test-isolation"
|
||||
"check:wasm"
|
||||
"check:admin-build"
|
||||
|
||||
+81
-8
@@ -45,7 +45,7 @@ import {
|
||||
buildBasenameIndex,
|
||||
queryBasenameIndex,
|
||||
} from '../core/link-extraction.ts';
|
||||
import { isSourceUnchangedSinceSync } from '../core/git-head.ts';
|
||||
import { probeSourceGitState } from '../core/git-head.ts';
|
||||
// v0.41.32.0: remote staleness reads the stored newest_content_at column via
|
||||
// this pure comparator (no git subprocess on the HTTP MCP doctor path).
|
||||
import { lagFromContentMs } from '../core/source-health.ts';
|
||||
@@ -58,6 +58,7 @@ import { isUndefinedColumnError } from '../core/utils.ts';
|
||||
// drift from what search actually filters.
|
||||
import { resolveHardExcludes, DEFAULT_HARD_EXCLUDES } from '../core/search/source-boost.ts';
|
||||
import { escapeLikePattern, buildVisibilityClause } from '../core/search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from '../core/extraction-review.ts';
|
||||
import { hnswIndexExpected, hnswMaxDimsForType } from '../core/vector-index.ts';
|
||||
|
||||
export interface Check {
|
||||
@@ -3625,6 +3626,52 @@ export async function checkLinksExtractionLag(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #160 — unverified_extractions doctor check.
|
||||
*
|
||||
* The extraction quarantine lane parks auto-extracted entity stubs
|
||||
* (frontmatter `provenance: 'auto-extracted'` + `status: 'unverified'`)
|
||||
* until the owner promotes or rejects them. A queue nobody reviews decays
|
||||
* into invisible clutter, so this check counts stubs older than N days
|
||||
* (default 7) and nudges toward the review surface. Exported for direct
|
||||
* testing (mirrors checkLinksExtractionLag).
|
||||
*/
|
||||
export async function checkUnverifiedExtractions(
|
||||
engine: BrainEngine,
|
||||
opts?: { sourceId?: string; days?: number },
|
||||
): Promise<Check> {
|
||||
const name = 'unverified_extractions';
|
||||
const days = opts?.days ?? 7;
|
||||
const sourceId = opts?.sourceId;
|
||||
try {
|
||||
const params: unknown[] = [String(days)];
|
||||
let srcClause = '';
|
||||
if (sourceId) {
|
||||
params.push(sourceId);
|
||||
srcClause = 'AND p.source_id = $2';
|
||||
}
|
||||
const rows = await engine.executeRaw<{ n: string | number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND ${unverifiedExtractionFragment('p')}
|
||||
AND p.created_at < now() - ($1 || ' days')::interval
|
||||
${srcClause}`,
|
||||
params,
|
||||
);
|
||||
const n = Number(rows[0]?.n ?? 0);
|
||||
return {
|
||||
name,
|
||||
status: n > 0 ? 'warn' : 'ok',
|
||||
message: n > 0
|
||||
? `${n} unverified auto-extracted entity stub(s) older than ${days} days awaiting review. List with 'gbrain extraction-pending'; promote/reject with 'gbrain extraction-review <promote|reject> --slugs <slug,...>'.`
|
||||
: 'No stale unverified extraction stubs',
|
||||
details: { count: n, days, source_id: sourceId ?? null },
|
||||
};
|
||||
} catch (e) {
|
||||
return { name, status: 'warn', message: `Could not check unverified_extractions: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1678 — extract_atoms_backlog doctor check.
|
||||
*
|
||||
@@ -4020,29 +4067,51 @@ export async function checkSyncFreshness(
|
||||
// All four must hold; otherwise fall through to the time-based check.
|
||||
// The chunker version match is computed here (not in the helper)
|
||||
// because it depends on engine state, not git state.
|
||||
//
|
||||
// Clone-unavailable fallback: on stateless deploys (Docker on EB /
|
||||
// K8s / Fly — the platforms the cloud recipes produce), a container
|
||||
// restart wipes `local_path` and each clone is only re-materialized
|
||||
// when that source's next sync job runs. Until then the HEAD probe
|
||||
// cannot run at all ('unavailable'), which previously fell through to
|
||||
// raw wall-clock age — and since a no-op sync doesn't advance
|
||||
// `last_sync_at`, every QUIET source read as stale/FAIL after a
|
||||
// restart (score-sinking alert storm; observed live: 16-source brain,
|
||||
// 12 clones gone after a config-update restart, doctor 70→30).
|
||||
// 'unavailable' + chunker match now reuses the v0.41.32.0 REMOTE lag
|
||||
// signal (newest_content_at) below — DB-only, no subprocess, and it
|
||||
// still reports staleness whenever content really is newer than the
|
||||
// last sync. 'changed' (readable clone with real work) keeps
|
||||
// wall-clock exactly as before, and a chunker mismatch is never
|
||||
// masked (D7): it disables the fallback too.
|
||||
let cloneUnavailable = false;
|
||||
if (localOnly) {
|
||||
const gitUnchanged = isSourceUnchangedSinceSync(
|
||||
const gitState = probeSourceGitState(
|
||||
source.local_path,
|
||||
source.last_commit,
|
||||
{ requireCleanWorkingTree: 'ignore-untracked' },
|
||||
);
|
||||
const chunkerMatch = source.chunker_version === currentChunkerVersion;
|
||||
if (gitUnchanged && chunkerMatch) {
|
||||
if (gitState === 'unchanged' && chunkerMatch) {
|
||||
unchanged_count++;
|
||||
continue;
|
||||
}
|
||||
cloneUnavailable = gitState === 'unavailable' && chunkerMatch;
|
||||
}
|
||||
|
||||
// v0.41.32.0: REMOTE path (doctorReportRemote, !localOnly) computes lag
|
||||
// from the stored newest_content_at column — NO git subprocess on a
|
||||
// DB-supplied local_path (preserves the v0.41.27.0 trust boundary). A
|
||||
// quiet repo whose newest commit predates its last sync reports 0; NULL
|
||||
// column → wall-clock fallback. LOCAL fall-through keeps wall-clock: the
|
||||
// short-circuit already failed, so the source genuinely has work and
|
||||
// "hours since last sync" is the right staleness measure. The `ageMs < 0`
|
||||
// skew check above still runs on raw wall-clock for both paths (A1).
|
||||
// column → wall-clock fallback. LOCAL fall-through keeps wall-clock when
|
||||
// the clone is READABLE: the short-circuit failed on real evidence
|
||||
// (HEAD moved / dirty tree), so the source genuinely has work and
|
||||
// "hours since last sync" is the right staleness measure. A local clone
|
||||
// that is UNAVAILABLE (not yet re-materialized, see above) carries no
|
||||
// evidence either way, so it borrows this same DB-only lag. The
|
||||
// `ageMs < 0` skew check above still runs on raw wall-clock for both
|
||||
// paths (A1).
|
||||
let thresholdAgeMs = ageMs;
|
||||
if (!localOnly) {
|
||||
if (!localOnly || cloneUnavailable) {
|
||||
const contentMs = source.newest_content_at
|
||||
? new Date(source.newest_content_at).getTime()
|
||||
: null;
|
||||
@@ -6905,6 +6974,10 @@ export async function buildChecks(
|
||||
checks.push({ name: 'flagged_pages', status: 'ok', message: `Skipped (${msg})` });
|
||||
}
|
||||
|
||||
// issue #160: extraction quarantine lane review nudge.
|
||||
progress.heartbeat('unverified_extractions');
|
||||
checks.push(await checkUnverifiedExtractions(engine, { sourceId: orphanRatioSourceId }));
|
||||
|
||||
// 11a. Frontmatter integrity (v0.22.4, hardened in v0.38.2.0).
|
||||
// scanBrainSources walks every registered source's local_path on disk
|
||||
// (not from the DB), invoking parseMarkdown(..., {validate:true}) per
|
||||
|
||||
@@ -149,6 +149,40 @@ export const ALLOWED_TYPES = [
|
||||
] as const;
|
||||
export type AllowedType = (typeof ALLOWED_TYPES)[number];
|
||||
|
||||
/**
|
||||
* Granular collector page-types that alias into each canonical conversation
|
||||
* bucket. The v2 type-consolidation pack retypes these to the canonical names
|
||||
* (`slack-dm-day`/`slack-thread` → `slack`, `email-digest` → `email`), but a
|
||||
* brain that hasn't run that pack still carries the collector's granular types
|
||||
* in `pages.type`. Without this expansion, `listPages({ type: 'slack' })`
|
||||
* matches zero rows on such brains and the whole comms corpus is silently
|
||||
* skipped (facts stay empty → `find_trajectory` returns nothing). The canonical
|
||||
* name is always included first so consolidated brains keep working unchanged.
|
||||
*/
|
||||
export const ALLOWED_TYPE_ALIASES: Record<AllowedType, readonly string[]> = {
|
||||
conversation: ['conversation'],
|
||||
meeting: ['meeting'],
|
||||
slack: ['slack', 'slack-dm-day', 'slack-thread'],
|
||||
email: ['email', 'email-digest'],
|
||||
imessage: ['imessage'],
|
||||
'imessage-daily': ['imessage-daily'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Expand the requested logical types to the concrete `pages.type` values to
|
||||
* enumerate, canonical-first and de-duplicated. Unknown types pass through
|
||||
* unchanged so an explicit override is never dropped.
|
||||
*/
|
||||
export function pageTypesForAllowed(types: readonly AllowedType[]): string[] {
|
||||
const out: string[] = [];
|
||||
for (const t of types) {
|
||||
for (const concrete of ALLOWED_TYPE_ALIASES[t] ?? [t]) {
|
||||
if (!out.includes(concrete)) out.push(concrete);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pagination batch size for listPages enumeration. Per-batch memory
|
||||
* worst case = BATCH × MAX_PAGE_BODY_BYTES = 250MB at default 10
|
||||
@@ -1264,13 +1298,18 @@ export async function runExtractConversationFactsCore(
|
||||
}
|
||||
};
|
||||
|
||||
// Expand logical types (conversation/meeting/slack/email) to the concrete
|
||||
// `pages.type` values to enumerate, so brains on the granular collector
|
||||
// types are not silently skipped (see ALLOWED_TYPE_ALIASES).
|
||||
const concreteTypes = pageTypesForAllowed(types);
|
||||
|
||||
if (opts.slug) {
|
||||
const page = await engine.getPage(opts.slug, { sourceId });
|
||||
if (!page) {
|
||||
result.pages_skipped_disappeared++;
|
||||
return;
|
||||
}
|
||||
if (!types.includes(page.type as AllowedType)) {
|
||||
if (!concreteTypes.includes(page.type)) {
|
||||
result.pages_skipped++;
|
||||
return;
|
||||
}
|
||||
@@ -1284,7 +1323,7 @@ export async function runExtractConversationFactsCore(
|
||||
// honors AbortSignal at each claim boundary and threads
|
||||
// BudgetExhausted abort (D13) automatically.
|
||||
let processedPagesCount = 0;
|
||||
pageLoop: for (const type of types) {
|
||||
pageLoop: for (const type of concreteTypes) {
|
||||
let offset = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
|
||||
+23
-4
@@ -35,7 +35,7 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput } from '../core/en
|
||||
import type { PageType } from '../core/types.ts';
|
||||
import { parseMarkdown } from '../core/markdown.ts';
|
||||
import {
|
||||
extractPageLinks, parseTimelineEntries, inferLinkType, makeResolver,
|
||||
extractPageLinks, parseTimelineEntries, deriveTimelineAnchor, inferLinkType, makeResolver,
|
||||
extractFrontmatterLinks, isGlobalBasenameEnabled, LINK_EXTRACTOR_VERSION_TS,
|
||||
WIKILINK_BASENAME_LINK_TYPE,
|
||||
buildBasenameIndex, queryBasenameIndex, stripCodeBlocks,
|
||||
@@ -749,6 +749,12 @@ export async function runExtract(engine: BrainEngine, args: string[]) {
|
||||
// v0.41.18.0 (A11, T8): --from-meetings extracts timeline entries from
|
||||
// meeting pages onto each discussed entity. Timeline subcommand only.
|
||||
const fromMeetings = args.includes('--from-meetings');
|
||||
// --infer-dates: for pages whose body has NO parseable timeline line, anchor
|
||||
// one entry at the page's computed effective_date (frontmatter / filename date,
|
||||
// never the updated_at fallback). Default OFF for back-compat — comms/calendar
|
||||
// brains opt in to populate timeline from slug/frontmatter dates. DB-source only
|
||||
// (needs the full Page.effective_date, which getPage projects).
|
||||
const inferDates = args.includes('--infer-dates');
|
||||
// v0.41.17.0 (T7, D9): --workers N parsed via the shared validator.
|
||||
// Honored on the fs-walk inner loops only; DB-source paths stay
|
||||
// serial in v0.41.17.0 (see ExtractOpts.workers doc).
|
||||
@@ -963,7 +969,7 @@ Status (v0.42):
|
||||
result.pages_processed = r.pages;
|
||||
}
|
||||
if (subcommand === 'timeline' || subcommand === 'all') {
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter });
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter, inferDates });
|
||||
result.timeline_entries_created = r.created;
|
||||
result.pages_processed = Math.max(result.pages_processed, r.pages);
|
||||
}
|
||||
@@ -1583,7 +1589,7 @@ async function extractTimelineFromDB(
|
||||
jsonMode: boolean,
|
||||
typeFilter: PageType | undefined,
|
||||
since: string | undefined,
|
||||
opts?: { sourceIdFilter?: string },
|
||||
opts?: { sourceIdFilter?: string; inferDates?: boolean },
|
||||
): Promise<{ created: number; pages: number }> {
|
||||
// v0.32.8: listAllPageRefs enumerates (slug, source_id) pairs so we can
|
||||
// thread sourceId to getPage and addTimelineEntriesBatch. Pre-fix used
|
||||
@@ -1592,6 +1598,7 @@ async function extractTimelineFromDB(
|
||||
// v0.37.7.0 #1204: when sourceIdFilter is set, scope the walk to one
|
||||
// source so federated brain users can extract per-source.
|
||||
const sourceIdFilter = opts?.sourceIdFilter;
|
||||
const inferDates = opts?.inferDates ?? false;
|
||||
const allRefs = sourceIdFilter
|
||||
? (await engine.listAllPageRefs()).filter(r => r.source_id === sourceIdFilter)
|
||||
: await engine.listAllPageRefs();
|
||||
@@ -1631,7 +1638,19 @@ async function extractTimelineFromDB(
|
||||
}
|
||||
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const entries = parseTimelineEntries(fullContent);
|
||||
let entries = parseTimelineEntries(fullContent);
|
||||
// --infer-dates: pages with no in-body timeline line but a trustworthy
|
||||
// content date (frontmatter / filename) get one anchor entry at that date.
|
||||
// Applied ONLY on the zero-entry path so it never shadows a real timeline.
|
||||
if (entries.length === 0 && inferDates) {
|
||||
const anchor = deriveTimelineAnchor({
|
||||
slug,
|
||||
title: page.title,
|
||||
effectiveDate: page.effective_date,
|
||||
effectiveDateSource: page.effective_date_source,
|
||||
});
|
||||
if (anchor) entries = [anchor];
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (dryRunSeen) {
|
||||
|
||||
+27
-2
@@ -143,6 +143,31 @@ export function resolveWorkerConcurrency(args: string[], env: NodeJS.ProcessEnv
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* #3026: the thin-client `list`/`get` branches receive jobs as parsed JSON
|
||||
* off the MCP wire, where every timestamp is an ISO string — but formatJob /
|
||||
* formatJobDetail (and the stalled-detection comparison) hold a Date
|
||||
* contract, hydrated locally by MinionQueue.rowToJob. Rehydrate once at the
|
||||
* unpack boundary so both paths hand the formatters real Dates. Exported for
|
||||
* unit tests.
|
||||
*/
|
||||
const JOB_DATE_FIELDS = [
|
||||
'created_at', 'updated_at', 'started_at', 'finished_at', 'lock_until', 'delay_until',
|
||||
] as const;
|
||||
|
||||
export function rehydrateJobDates<T>(job: T): T {
|
||||
if (!job || typeof job !== 'object') return job;
|
||||
const rec = job as { [k: string]: unknown };
|
||||
for (const field of JOB_DATE_FIELDS) {
|
||||
const v = rec[field];
|
||||
if (typeof v === 'string') {
|
||||
const d = new Date(v);
|
||||
if (!Number.isNaN(d.getTime())) rec[field] = d;
|
||||
}
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
function formatJob(job: MinionJob): string {
|
||||
const dur = job.finished_at && job.started_at
|
||||
? `${((job.finished_at.getTime() - job.started_at.getTime()) / 1000).toFixed(1)}s`
|
||||
@@ -496,7 +521,7 @@ HANDLER TYPES (built in)
|
||||
const raw = await callRemoteTool(cfg!, 'list_jobs', {
|
||||
status, queue: queueName, limit,
|
||||
}, { timeoutMs: 30_000 });
|
||||
jobs = unpackToolResult<MinionJob[]>(raw);
|
||||
jobs = unpackToolResult<MinionJob[]>(raw).map((j) => rehydrateJobDates(j));
|
||||
} else {
|
||||
try { await queue.ensureSchema(); }
|
||||
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
|
||||
@@ -525,7 +550,7 @@ HANDLER TYPES (built in)
|
||||
if (isThinClient(cfg)) {
|
||||
try {
|
||||
const raw = await callRemoteTool(cfg!, 'get_job', { id }, { timeoutMs: 30_000 });
|
||||
job = unpackToolResult<MinionJob | null>(raw);
|
||||
job = rehydrateJobDates(unpackToolResult<MinionJob | null>(raw));
|
||||
} catch (e) {
|
||||
// The remote op throws `invalid_params` on not-found; surface as
|
||||
// the same "Job not found" exit-1 the local path produces.
|
||||
|
||||
@@ -31,6 +31,11 @@ export const dashscope: Recipe = {
|
||||
// path. Conservative declaration so the gateway pre-splits before
|
||||
// hitting whatever undocumented server-side limit exists.
|
||||
max_batch_tokens: 8192,
|
||||
// DashScope's OpenAI-compat /embeddings endpoint rejects requests with
|
||||
// more than 10 input items (documented Model Studio cap). The gateway's
|
||||
// capBatchItems pre-split enforces this; max_batch_tokens above keeps
|
||||
// guarding aggregate token size. Concept from community PRs #2643/#2405.
|
||||
max_batch_items: 10,
|
||||
// text-embedding-v3 mixes English + CJK heavily; the tokenizer is
|
||||
// closer to Voyage density than OpenAI tiktoken for CJK-dominant
|
||||
// content. Conservative chars_per_token=2 leaves headroom.
|
||||
|
||||
+125
-11
@@ -40,6 +40,7 @@ export const LINKABLE_ENTITY_TYPES = ['person', 'company', 'organization', 'enti
|
||||
* types in.
|
||||
*/
|
||||
const MIN_NAME_LENGTH = 4;
|
||||
const MIN_CJK_NAME_LENGTH = 2;
|
||||
|
||||
/**
|
||||
* Built-in ignore list — common ambiguous tokens whose body-text mentions
|
||||
@@ -104,12 +105,12 @@ export interface FindMentionsOpts {
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs for every
|
||||
* `[a-zA-Z0-9]+` run, lowercased. Non-ASCII (CJK, accented) is
|
||||
* deliberately not tokenized in v1 — entity gazetteer is English-dominant
|
||||
* in production today. Widening to `\p{L}+` is a future option once a
|
||||
* real CJK entity catalog appears (filed under TODO-1 + a TODO for
|
||||
* Unicode-aware tokenization).
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs.
|
||||
*
|
||||
* ASCII: each `[a-zA-Z0-9]+` run is a single token, lowercased.
|
||||
* CJK: each CJK character (Chinese/Japanese/Korean) is an individual
|
||||
* token, lowercased. This allows the normal maximal-munch scan path
|
||||
* to reach CJK gazetteer entries without a separate substring pass.
|
||||
*
|
||||
* Possessive "Acme's" tokenizes as ['acme', 's'] (single-quote breaks the
|
||||
* run) — single-word "Acme" lookup succeeds at offset 0; the trailing 's'
|
||||
@@ -127,18 +128,129 @@ function tokenizeForScan(text: string): ScannedToken[] {
|
||||
const out: ScannedToken[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
|
||||
// Collect ASCII token spans first.
|
||||
const asciiSpans: Array<{ start: number; end: number }> = [];
|
||||
while ((m = TOKEN_RE.exec(text)) !== null) {
|
||||
out.push({ text: m[0].toLowerCase(), offset: m.index, length: m[0].length });
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length });
|
||||
}
|
||||
|
||||
// Walk character-by-character: emit ASCII tokens at their start positions,
|
||||
// then emit individual CJK characters for non-ASCII positions that fall
|
||||
// outside ASCII token spans.
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < text.length;) {
|
||||
const cp = text.codePointAt(i) ?? 0;
|
||||
const isCJK = (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af);
|
||||
|
||||
// Advance asciiIdx past any spans that end before or at i.
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) {
|
||||
asciiIdx++;
|
||||
}
|
||||
|
||||
// If position i is inside an ASCII token span, emit the full ASCII token
|
||||
// and jump past it.
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
const span = asciiSpans[asciiIdx]!;
|
||||
const token = text.slice(span.start, span.end);
|
||||
out.push({ text: token.toLowerCase(), offset: span.start, length: token.length });
|
||||
i = span.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// CJK: emit as individual character token.
|
||||
if (isCJK) {
|
||||
const charLen = cp > 0xffff ? 2 : 1; // surrogate pair
|
||||
const charStr = text.slice(i, i + charLen);
|
||||
out.push({ text: charStr.toLowerCase(), offset: i, length: charLen });
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function hasCJK(s: string): boolean {
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function cjkCharCount(s: string): number {
|
||||
let count = 0;
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a page title for gazetteer insertion.
|
||||
*
|
||||
* ASCII titles: standard `[a-zA-Z0-9]+` tokenization, lowercased.
|
||||
* CJK titles (no ASCII content): split into individual characters —
|
||||
* e.g. "纳瓦尔" → ["纳","瓦","尔"]. This allows normal multi-token
|
||||
* maximal-munch matching to work with character-level CJK tokens
|
||||
* produced by `tokenizeForScan`.
|
||||
* Mixed CJK+ASCII titles: ASCII parts tokenized normally, CJK parts
|
||||
* split into individual characters.
|
||||
*/
|
||||
function tokenizeTitle(title: string): string[] {
|
||||
const tokens: string[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) tokens.push(m[0].toLowerCase());
|
||||
return tokens;
|
||||
const hasAscii = TOKEN_RE.test(title);
|
||||
if (hasAscii) {
|
||||
// Mixed ASCII+CJK or pure ASCII: tokenize ASCII normally, then
|
||||
// append individual CJK characters in order.
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
const asciiSpans: Array<{ start: number; end: number; text: string }> = [];
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) {
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length, text: m[0].toLowerCase() });
|
||||
}
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < title.length;) {
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) asciiIdx++;
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
tokens.push(asciiSpans[asciiIdx]!.text);
|
||||
i = asciiSpans[asciiIdx]!.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
if (hasCJK(title[i]!)) {
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Pure CJK (no ASCII content): split into individual characters.
|
||||
if (hasCJK(title)) {
|
||||
for (let i = 0; i < title.length;) {
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Non-ASCII, non-CJK title (emoji, symbols, etc.) — empty set.
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -175,7 +287,9 @@ export async function buildGazetteer(
|
||||
|
||||
const gazetteer: Gazetteer = new Map();
|
||||
for (const row of rows) {
|
||||
if (!row.title || row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (!row.title) continue;
|
||||
if (!hasCJK(row.title) && row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (hasCJK(row.title) && cjkCharCount(row.title) < MIN_CJK_NAME_LENGTH) continue;
|
||||
if (ignoreSet.has(row.title) && !existingTitles.has(row.title)) continue;
|
||||
|
||||
const tokens = tokenizeTitle(row.title);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Block-format conversation normalizer.
|
||||
*
|
||||
* Some chat exports — notably the Slack collector gbrain's own ingestion
|
||||
* uses — emit a HEADER + indented-body BLOCK per message instead of the
|
||||
* single-line `**Name** (time): body` shape the built-in patterns
|
||||
* (`builtins.ts`) recognize:
|
||||
*
|
||||
* - **Theo** (Mon 11:18)
|
||||
* Hey everyone — quick update on the renewal.
|
||||
*
|
||||
* Second paragraph of the same message.
|
||||
* - **Juan** (Mon 11:20)
|
||||
* Reply body...
|
||||
*
|
||||
* None of the 14 line-oriented built-ins match this: a leading `- ` list
|
||||
* marker, a day-of-week + time with no trailing colon, and the message body on
|
||||
* the following indented lines. Result: `phase: 'no_match'`, zero messages,
|
||||
* and the whole comms corpus is silently un-extractable (facts stay empty →
|
||||
* `find_trajectory` returns nothing).
|
||||
*
|
||||
* This collapses each block into the canonical `**Name** (HH:MM): <body joined
|
||||
* to one line>` shape so the existing `bold-paren-time` pattern matches; the
|
||||
* per-message date fills in downstream via `fallbackDate` (the page date).
|
||||
*
|
||||
* STRICT no-op unless the block signature is present: the header regex requires
|
||||
* the paren-group to END the line (no inline `: body`), which is exactly what
|
||||
* the single-line patterns always produce — so feeding already-canonical
|
||||
* content through this function returns it unchanged.
|
||||
*/
|
||||
|
||||
// `- **Name** (Mon 11:18)` / `- **Name** (11:18 AM)` / `- **Name** (16:36)`.
|
||||
// Day-of-week optional; 12h/24h time; optional am/pm; the line ENDS at the
|
||||
// close paren (no inline `: body` — that is what distinguishes a block header
|
||||
// from the single-line `**Name** (time): body` patterns).
|
||||
const BLOCK_HEADER =
|
||||
/^\s*-\s+\*\*(.+?)\*\*\s+\((?:[A-Za-z]{2,9}\.?\s+)?(\d{1,2}):(\d{2})(?::\d{2})?\s*([AaPp][Mm])?\)\s*$/;
|
||||
|
||||
/** True when at least one line is a block-format message header. */
|
||||
export function looksLikeBlockConversation(body: string): boolean {
|
||||
for (const line of body.split('\n')) {
|
||||
if (BLOCK_HEADER.test(line)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function to24h(hour: number, ampm?: string): number {
|
||||
if (!ampm) return hour;
|
||||
const lower = ampm.toLowerCase();
|
||||
if (lower === 'pm' && hour < 12) return hour + 12;
|
||||
if (lower === 'am' && hour === 12) return 0;
|
||||
return hour;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse block-format messages into canonical single-line `**Name** (HH:MM):
|
||||
* body` lines. Returns `body` unchanged when no block header is present.
|
||||
*/
|
||||
export function normalizeBlockConversation(body: string): string {
|
||||
if (!looksLikeBlockConversation(body)) return body;
|
||||
|
||||
const lines = body.split('\n');
|
||||
const out: string[] = [];
|
||||
let current: { name: string; time: string } | null = null;
|
||||
let bodyParts: string[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (current) {
|
||||
const text = bodyParts.join(' ').replace(/\s+/g, ' ').trim();
|
||||
out.push(`**${current.name}** (${current.time}): ${text}`);
|
||||
}
|
||||
current = null;
|
||||
bodyParts = [];
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const m = BLOCK_HEADER.exec(line);
|
||||
if (m) {
|
||||
flush();
|
||||
const hour = to24h(parseInt(m[2], 10), m[4]);
|
||||
const time = `${String(hour).padStart(2, '0')}:${m[3]}`;
|
||||
current = { name: m[1].trim(), time };
|
||||
} else if (current) {
|
||||
// Body line of the current message. Drop blank lines; keep the rest.
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) bodyParts.push(trimmed);
|
||||
}
|
||||
// Lines before the first header (page title, leading blanks) are dropped —
|
||||
// they never matched a pattern anyway.
|
||||
}
|
||||
flush();
|
||||
|
||||
return out.length > 0 ? out.join('\n') : body;
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
BUILTIN_PATTERNS,
|
||||
cleanSpeaker,
|
||||
} from './builtins.ts';
|
||||
import { normalizeBlockConversation } from './normalize-block.ts';
|
||||
import type {
|
||||
DateContext,
|
||||
MatchedMessage,
|
||||
@@ -473,6 +474,12 @@ export function parseConversation(
|
||||
return { messages: [], phase: 'no_match' };
|
||||
}
|
||||
|
||||
// Pre-pass: collapse block-format chat exports (header + indented body, e.g.
|
||||
// the Slack collector's `- **Name** (Mon 11:18)\n body…`) into the canonical
|
||||
// single-line shape the built-in patterns recognize. Strict no-op when no
|
||||
// block header is present, so already-canonical content is untouched.
|
||||
body = normalizeBlockConversation(body);
|
||||
|
||||
const dateCtx = deriveDateContext(opts);
|
||||
|
||||
// Assemble candidate pool: built-ins (minus disabled) + user patterns.
|
||||
|
||||
@@ -112,6 +112,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'takes_weight_grid',
|
||||
'timeline_coverage',
|
||||
'unified_multimodal_coverage',
|
||||
'unverified_extractions',
|
||||
'voice_gate_health',
|
||||
]);
|
||||
|
||||
|
||||
@@ -1341,6 +1341,16 @@ export interface BrainEngine {
|
||||
getContentFlagsByPageIds(
|
||||
pageIds: number[],
|
||||
): Promise<Map<number, { reason: string; detail: string }>>;
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160): for a list of page_ids, return
|
||||
* the subset that are unverified auto-extracted entity stubs (frontmatter
|
||||
* `provenance: 'auto-extracted'` + `status: 'unverified'`). Used by hybrid
|
||||
* search to stamp `SearchResult.unverified` pre-fusion so the fusion-level
|
||||
* compiled-truth boost skips them. Single SQL query, not N+1. Empty input
|
||||
* → empty set (no query). SQL predicate is the shared
|
||||
* `unverifiedExtractionFragment` (src/core/extraction-review.ts).
|
||||
*/
|
||||
getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>>;
|
||||
/**
|
||||
* v0.27.0: for a list of slugs, return their updated_at timestamps (or created_at fallback).
|
||||
* Used by hybrid search recency boost. Single SQL query, not N+1.
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { waitForCapacity } from './backoff.ts';
|
||||
import { quarantineMarkers } from './extraction-review.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -28,9 +29,32 @@ export interface EnrichmentRequest {
|
||||
tier?: 1 | 2 | 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trust options for the enrichment write path (issue #160).
|
||||
*
|
||||
* `trusted: true` — the input text comes from the machine owner via the
|
||||
* trusted local CLI (ctx.remote === false) AND the caller passed an explicit
|
||||
* opt-in flag. Stubs write direct as authoritative entity pages.
|
||||
*
|
||||
* Anything else (undefined, false, absent) is UNTRUSTED — fail-closed,
|
||||
* mirroring the OperationContext.remote invariant ("anything not strictly
|
||||
* false is remote"). Created stubs land in the quarantine lane: frontmatter
|
||||
* `provenance: 'auto-extracted'` + `status: 'unverified'`. They are excluded
|
||||
* from authoritative retrieval boosts and wait in the review queue
|
||||
* (`extraction_pending` / `extraction_review` ops) until the owner promotes
|
||||
* or rejects them.
|
||||
*/
|
||||
export interface EnrichmentTrustOptions {
|
||||
trusted?: boolean;
|
||||
/** Source to read/write in (multi-source brains). Omitted → engine default. */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface EnrichmentResult {
|
||||
slug: string;
|
||||
action: 'created' | 'updated' | 'skipped';
|
||||
/** True when the created stub landed in the quarantine lane (issue #160). */
|
||||
quarantined?: boolean;
|
||||
tier: 1 | 2 | 3;
|
||||
backlinkCreated: boolean;
|
||||
timelineAdded: boolean;
|
||||
@@ -72,11 +96,15 @@ export function entityPagePath(name: string, type: 'person' | 'company'): string
|
||||
export async function enrichEntity(
|
||||
engine: BrainEngine,
|
||||
request: EnrichmentRequest,
|
||||
opts?: EnrichmentTrustOptions,
|
||||
): Promise<EnrichmentResult> {
|
||||
const slug = slugifyEntity(request.entityName, request.entityType);
|
||||
// Fail-closed: only an explicit `trusted: true` writes authoritative pages.
|
||||
const trusted = opts?.trusted === true;
|
||||
const scope = opts?.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
|
||||
// 1. Count existing mentions for tier auto-escalation
|
||||
const { mentionCount, mentionSources } = await countMentions(engine, request.entityName);
|
||||
const { mentionCount, mentionSources } = await countMentions(engine, request.entityName, opts?.sourceId);
|
||||
|
||||
// 2. Determine tier (auto-escalate based on mentions)
|
||||
const suggestedTier = suggestTier(mentionCount, mentionSources, request.context);
|
||||
@@ -84,7 +112,7 @@ export async function enrichEntity(
|
||||
const tierEscalated = suggestedTier < (request.tier || 3); // lower tier number = higher importance
|
||||
|
||||
// 3. Check if entity page exists
|
||||
const existingPage = await engine.getPage(slug);
|
||||
const existingPage = await engine.getPage(slug, scope);
|
||||
let action: 'created' | 'updated' | 'skipped';
|
||||
|
||||
if (existingPage) {
|
||||
@@ -104,8 +132,11 @@ export async function enrichEntity(
|
||||
created: new Date().toISOString().split('T')[0],
|
||||
source: request.sourceSlug,
|
||||
tier,
|
||||
// issue #160 quarantine lane: stubs extracted from untrusted input
|
||||
// carry provenance + unverified markers until the owner reviews them.
|
||||
...(trusted ? {} : quarantineMarkers()),
|
||||
},
|
||||
});
|
||||
}, scope);
|
||||
action = 'created';
|
||||
}
|
||||
|
||||
@@ -116,7 +147,7 @@ export async function enrichEntity(
|
||||
date: new Date().toISOString().split('T')[0] ?? '',
|
||||
summary: `Referenced in [${request.sourceSlug}](${request.sourceSlug}) — ${request.context}`,
|
||||
source: request.sourceSlug,
|
||||
});
|
||||
}, scope);
|
||||
timelineAdded = true;
|
||||
} catch {
|
||||
// Timeline add failed (page might not support it)
|
||||
@@ -125,7 +156,7 @@ export async function enrichEntity(
|
||||
// 5. Add backlink from entity to source
|
||||
let backlinkCreated = false;
|
||||
try {
|
||||
await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdown
|
||||
await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`, undefined, undefined, undefined, undefined, opts?.sourceId ? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId } : undefined); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdown
|
||||
backlinkCreated = true;
|
||||
} catch {
|
||||
// Link might already exist
|
||||
@@ -134,6 +165,7 @@ export async function enrichEntity(
|
||||
return {
|
||||
slug,
|
||||
action,
|
||||
...(action === 'created' && !trusted ? { quarantined: true } : {}),
|
||||
tier,
|
||||
backlinkCreated,
|
||||
timelineAdded,
|
||||
@@ -152,14 +184,14 @@ export async function enrichEntity(
|
||||
export async function enrichEntities(
|
||||
engine: BrainEngine,
|
||||
requests: EnrichmentRequest[],
|
||||
config?: { throttle?: boolean; onProgress?: (done: number, total: number, name: string) => void },
|
||||
config?: { throttle?: boolean; onProgress?: (done: number, total: number, name: string) => void } & EnrichmentTrustOptions,
|
||||
): Promise<EnrichmentResult[]> {
|
||||
const results: EnrichmentResult[] = [];
|
||||
for (const req of requests) {
|
||||
if (config?.throttle !== false) {
|
||||
await waitForCapacity({ maxAttempts: 5 }); // shorter timeout for batch items
|
||||
}
|
||||
const result = await enrichEntity(engine, req);
|
||||
const result = await enrichEntity(engine, req, { trusted: config?.trusted, sourceId: config?.sourceId });
|
||||
results.push(result);
|
||||
config?.onProgress?.(results.length, requests.length, req.entityName);
|
||||
}
|
||||
@@ -175,8 +207,11 @@ export async function extractAndEnrich(
|
||||
engine: BrainEngine,
|
||||
text: string,
|
||||
sourceSlug: string,
|
||||
opts?: EnrichmentTrustOptions & { throttle?: boolean; maxEntities?: number },
|
||||
): Promise<EnrichmentResult[]> {
|
||||
const entities = extractEntities(text);
|
||||
// Bounded by default (#160 hardening): the greedy regex on a large paste
|
||||
// can produce thousands of hits; each enrichment is several DB round-trips.
|
||||
const entities = extractEntities(text).slice(0, opts?.maxEntities ?? 200);
|
||||
if (entities.length === 0) return [];
|
||||
|
||||
const requests: EnrichmentRequest[] = entities.map(e => ({
|
||||
@@ -186,7 +221,7 @@ export async function extractAndEnrich(
|
||||
sourceSlug,
|
||||
}));
|
||||
|
||||
return enrichEntities(engine, requests);
|
||||
return enrichEntities(engine, requests, { trusted: opts?.trusted, sourceId: opts?.sourceId, throttle: opts?.throttle });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -197,9 +232,10 @@ export async function extractAndEnrich(
|
||||
async function countMentions(
|
||||
engine: BrainEngine,
|
||||
entityName: string,
|
||||
sourceId?: string,
|
||||
): Promise<{ mentionCount: number; mentionSources: string[] }> {
|
||||
try {
|
||||
const results = await engine.searchKeyword(entityName, { limit: 100 });
|
||||
const results = await engine.searchKeyword(entityName, { limit: 100, ...(sourceId ? { sourceId } : {}) });
|
||||
// Derive sources from slug prefixes since SearchResult has no metadata.skill
|
||||
const sources = new Set<string>();
|
||||
for (const r of results) {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160).
|
||||
*
|
||||
* `extractAndEnrich` regex-extracts entity names from arbitrary ingested text
|
||||
* and creates `people/{slug}` / `companies/{slug}` stub pages. When the input
|
||||
* text comes from an untrusted channel (anything that is not the trusted local
|
||||
* CLI with an explicit opt-in), those stubs must NOT enter the brain as
|
||||
* authoritative entity pages. Instead they land in the quarantine lane:
|
||||
* ordinary pages carrying two frontmatter markers —
|
||||
*
|
||||
* provenance: 'auto-extracted' — HOW the page came to exist
|
||||
* status: 'unverified' — the owner has not reviewed it yet
|
||||
*
|
||||
* Consequences of the markers (each enforced at its own site):
|
||||
* - Search: unverified stubs are excluded from the compiled-truth authority
|
||||
* boost (they rank as ordinary content) and results carry
|
||||
* `unverified: true` so agents can label the provenance.
|
||||
* - Review: `extraction_pending` lists them; `extraction_review` promotes
|
||||
* (status → 'verified', provenance kept for audit) or rejects
|
||||
* (soft-delete) in batch. Promotion is local-owner-only.
|
||||
* - Doctor: counts unverified stubs older than N days as a review nudge.
|
||||
*
|
||||
* Fail-closed trust rule (mirrors OperationContext.remote): only an explicit
|
||||
* `trusted: true` writes direct; undefined/false/anything-else quarantines.
|
||||
*
|
||||
* Known scope (deliberate, documented — not gaps discovered later):
|
||||
* - CREATE-path only. The enrichment UPDATE path (timeline append + edge
|
||||
* onto an EXISTING page when a slug collides) is the separately-tracked
|
||||
* slug-collision finding referenced in issue #160; this lane does not
|
||||
* gate it.
|
||||
* - The markers are ordinary frontmatter keys, not put_page-strip-listed
|
||||
* (#1699). A caller holding generic remote put_page write scope can
|
||||
* rewrite a stub without them — but that caller can author an unmarked
|
||||
* people/ page directly anyway, so stripping here adds no privilege.
|
||||
* The promotion OP surface (extraction_review) is what stays owner-only.
|
||||
*
|
||||
* Sibling of `src/core/quarantine.ts` / `src/core/embed-skip.ts` — same
|
||||
* marker-as-frontmatter-JSONB pattern, same "SQL fragment lives next to the
|
||||
* marker key so they can never drift" rule. No schema migration needed.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Marker keys + values (stable contract)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const EXTRACTION_PROVENANCE_KEY = 'provenance';
|
||||
export const EXTRACTION_STATUS_KEY = 'status';
|
||||
|
||||
export const PROVENANCE_AUTO_EXTRACTED = 'auto-extracted';
|
||||
export const STATUS_UNVERIFIED = 'unverified';
|
||||
export const STATUS_VERIFIED = 'verified';
|
||||
|
||||
/** Frontmatter markers to spread onto a quarantined stub at create time. */
|
||||
export function quarantineMarkers(): Record<string, string> {
|
||||
return {
|
||||
[EXTRACTION_PROVENANCE_KEY]: PROVENANCE_AUTO_EXTRACTED,
|
||||
[EXTRACTION_STATUS_KEY]: STATUS_UNVERIFIED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* JS-side predicate: true only when BOTH markers match. Requiring the pair
|
||||
* means user pages that happen to carry their own `status` or `provenance`
|
||||
* frontmatter are never captured by the review lane.
|
||||
*/
|
||||
export function isUnverifiedExtraction(
|
||||
frontmatter: Record<string, unknown> | null | undefined,
|
||||
): boolean {
|
||||
if (!frontmatter) return false;
|
||||
return (
|
||||
frontmatter[EXTRACTION_PROVENANCE_KEY] === PROVENANCE_AUTO_EXTRACTED &&
|
||||
frontmatter[EXTRACTION_STATUS_KEY] === STATUS_UNVERIFIED
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment matching unverified auto-extracted stubs, parameterized on the
|
||||
* page-table alias. Single source of truth for every SQL-side consumer
|
||||
* (extraction_pending list, doctor count) so the filter and the marker keys
|
||||
* can never drift. `pageAlias` is engine-supplied (never user input).
|
||||
* JSONB `->>` works identically on Postgres and PGLite (PostgreSQL-in-WASM).
|
||||
*/
|
||||
export function unverifiedExtractionFragment(pageAlias: string): string {
|
||||
return (
|
||||
`(COALESCE(${pageAlias}.frontmatter, '{}'::jsonb) ->> '${EXTRACTION_PROVENANCE_KEY}') = '${PROVENANCE_AUTO_EXTRACTED}'` +
|
||||
` AND (COALESCE(${pageAlias}.frontmatter, '{}'::jsonb) ->> '${EXTRACTION_STATUS_KEY}') = '${STATUS_UNVERIFIED}'`
|
||||
);
|
||||
}
|
||||
+55
-13
@@ -96,29 +96,71 @@ export interface GitFreshnessOpts {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true iff `localPath` is a git repo whose current HEAD matches
|
||||
* `lastCommit`, AND (when `requireCleanWorkingTree`) the working tree
|
||||
* is clean.
|
||||
* Three-state git probe verdict for a federated source clone.
|
||||
*
|
||||
* - `'unchanged'`: HEAD matches `last_commit` (and, when requested, the
|
||||
* working tree is clean). Sync has nothing to do.
|
||||
* - `'changed'`: the clone is readable but HEAD moved, the tree is
|
||||
* dirty, or the DB never recorded a `last_commit` —
|
||||
* sync genuinely has (or may have) work.
|
||||
* - `'unavailable'`: the HEAD probe itself could not run — the clone
|
||||
* directory is missing, not a git repo, or git errored.
|
||||
* On stateless deploys (containers on EB / K8s / Fly,
|
||||
* where `local_path` dies with the filesystem and is
|
||||
* lazily re-materialized by the next per-source sync)
|
||||
* this is a NORMAL steady state for quiet sources, not
|
||||
* evidence of pending work. Callers can fall back to a
|
||||
* DB-only freshness signal instead of wall-clock age.
|
||||
*/
|
||||
export type SourceGitState = 'unchanged' | 'changed' | 'unavailable';
|
||||
|
||||
/**
|
||||
* Probe a source clone and classify it (see `SourceGitState`).
|
||||
*
|
||||
* This is NOT a full mirror of `gbrain sync`'s "do work?" predicate.
|
||||
* Chunker-version match is computed by the caller because it depends on
|
||||
* engine state (`sources.chunker_version` vs `CURRENT_CHUNKER_VERSION`).
|
||||
* See `src/commands/doctor.ts:checkSyncFreshness` for the AND
|
||||
* combination at the call site.
|
||||
*
|
||||
* NULL-input guard stays first: a NULL `last_commit` (legacy row) returns
|
||||
* `'changed'` WITHOUT running the head probe — same short-circuit contract
|
||||
* `isSourceUnchangedSinceSync` always had (pinned by doctor.test.ts case 4).
|
||||
*/
|
||||
export function probeSourceGitState(
|
||||
localPath: string | null | undefined,
|
||||
lastCommit: string | null | undefined,
|
||||
opts?: GitFreshnessOpts,
|
||||
): SourceGitState {
|
||||
if (!localPath || !lastCommit) return 'changed';
|
||||
const head = _headProbe(localPath);
|
||||
if (head === null) return 'unavailable';
|
||||
if (head !== lastCommit) return 'changed';
|
||||
if (opts?.requireCleanWorkingTree) {
|
||||
const ignoreUntracked = opts.requireCleanWorkingTree === 'ignore-untracked';
|
||||
const isClean = _cleanProbe(localPath, ignoreUntracked);
|
||||
// null (probe error) AND false (known dirty) both fail the gate. A clean
|
||||
// probe error with a READABLE head is not classified 'unavailable' —
|
||||
// fail toward "may have work" so the gate can only relax, never mask.
|
||||
if (isClean !== true) return 'changed';
|
||||
}
|
||||
return 'unchanged';
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true iff `localPath` is a git repo whose current HEAD matches
|
||||
* `lastCommit`, AND (when `requireCleanWorkingTree`) the working tree
|
||||
* is clean.
|
||||
*
|
||||
* Boolean façade over `probeSourceGitState` — `'unavailable'` and
|
||||
* `'changed'` both collapse to `false`, preserving the v0.41.27.0
|
||||
* fail-open contract for callers that only care about the short-circuit
|
||||
* (`src/core/source-health.ts`).
|
||||
*/
|
||||
export function isSourceUnchangedSinceSync(
|
||||
localPath: string | null | undefined,
|
||||
lastCommit: string | null | undefined,
|
||||
opts?: GitFreshnessOpts,
|
||||
): boolean {
|
||||
if (!localPath || !lastCommit) return false;
|
||||
const head = _headProbe(localPath);
|
||||
if (head === null || head !== lastCommit) return false;
|
||||
if (opts?.requireCleanWorkingTree) {
|
||||
const ignoreUntracked = opts.requireCleanWorkingTree === 'ignore-untracked';
|
||||
const isClean = _cleanProbe(localPath, ignoreUntracked);
|
||||
// null (probe error) AND false (known dirty) both fail the gate.
|
||||
if (isClean !== true) return false;
|
||||
}
|
||||
return true;
|
||||
return probeSourceGitState(localPath, lastCommit, opts) === 'unchanged';
|
||||
}
|
||||
|
||||
+75
-11
@@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import type { PageType, EffectiveDateSource } from './types.ts';
|
||||
import { ensureWellFormed } from './text-safe.ts';
|
||||
|
||||
/**
|
||||
@@ -671,6 +671,17 @@ const FOUNDED_RE = /\b(?:founded|co-?founded|started the company|incorporated|fo
|
||||
// "security advisor to|at", "product advisor to|at", "industry advisor".
|
||||
const ADVISES_RE = /\b(?:advises|advised|advisor (?:to|at|for|of)|advisory (?:board|role|position|capacity|engagement|partnership|contract|relationship|work)|board advisor|on .{0,20} advisory board|joined .{0,20} advisory board|in an? advisory (?:capacity|role|position)|as an? (?:advisor|security advisor|technical advisor|strategic advisor|industry advisor|product advisor|board advisor|senior advisor)|(?:strategic|technical|security|product|industry|senior|board) advisor (?:to|at|for|of)|consults for|consulting role (?:at|with))\b/i;
|
||||
|
||||
// Chinese link type patterns for CJK entity mentions.
|
||||
// NOTE: These patterns are Chinese-only (zh). Japanese and Korean link
|
||||
// type extraction is not yet implemented. Entity NAME extraction in
|
||||
// by-mention.ts covers all three scripts (CJK = Chinese/Japanese/Korean)
|
||||
// via Unicode-aware tokenization.
|
||||
const ZH_FOUNDED_RE = /(?:创立|创办|成立|创建|建立|开创|发起)(?:了|的)/;
|
||||
const ZH_INVESTED_RE = /(?:投资|入股|融资|注资|参股)(?:了|的|了?于)/;
|
||||
const ZH_ADVISES_RE = /(?:顾问|咨询|指导)(?:了|的)?/;
|
||||
const ZH_WORKS_AT_RE = /(?:任职|就职|担任|供职|在.{0,10}(?:工作|上班|负责))(?:于|在|的)?/;
|
||||
const ZH_CITED_RE = /(?:引用|援引|提到|提及|转述|摘录)(?:了|的|自)?/;
|
||||
|
||||
// Page-role detection: if the source page describes a partner/investor at
|
||||
// page level, that's a strong prior for outbound company refs being
|
||||
// invested_in even when per-edge context lacks explicit investment verbs.
|
||||
@@ -724,6 +735,12 @@ export function inferLinkType(pageType: PageType, context: string, globalContext
|
||||
if (INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ADVISES_RE.test(context)) return 'advises';
|
||||
if (WORKS_AT_RE.test(context)) return 'works_at';
|
||||
// Chinese link type patterns
|
||||
if (ZH_FOUNDED_RE.test(context)) return 'founded';
|
||||
if (ZH_INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ZH_ADVISES_RE.test(context)) return 'advises';
|
||||
if (ZH_WORKS_AT_RE.test(context)) return 'works_at';
|
||||
if (ZH_CITED_RE.test(context)) return 'cited';
|
||||
// Page-role prior: only fires for person -> company links. Concept pages
|
||||
// about VC topics naturally contain "venture capital" in their text, but
|
||||
// their company refs are mentions, not investments. Partner pages mentioning
|
||||
@@ -1174,6 +1191,10 @@ export interface TimelineCandidate {
|
||||
// Match: `- **YYYY-MM-DD** | summary` or `- **YYYY-MM-DD** -- summary`
|
||||
// or `- **YYYY-MM-DD** - summary` or just `**YYYY-MM-DD** | summary`.
|
||||
const TIMELINE_LINE_RE = /^\s*-?\s*\*\*(\d{4}-\d{2}-\d{2})\*\*\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
// Chinese date lines: `- 2020年1月2日 | summary` (bold optional). Requires the
|
||||
// 年/月 markers so plain ASCII `- 2020-01-02 - text` does NOT match — non-bold
|
||||
// ASCII dates were never timeline entries and must stay that way.
|
||||
const TIMELINE_LINE_RE_CN = /^\s*-?\s*(?:\*\*)?(\d{4})年(\d{1,2})月(\d{1,2})日?(?:\*\*)?\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
|
||||
/**
|
||||
* Parse timeline entries from content. Looks at:
|
||||
@@ -1190,18 +1211,21 @@ export function parseTimelineEntries(content: string): TimelineCandidate[] {
|
||||
|
||||
let i = 0;
|
||||
while (i < lines.length) {
|
||||
// Try English format first, then Chinese
|
||||
const m = TIMELINE_LINE_RE.exec(lines[i]);
|
||||
if (!m) {
|
||||
i++;
|
||||
continue;
|
||||
let date: string;
|
||||
let summary: string;
|
||||
if (m) {
|
||||
date = m[1];
|
||||
summary = m[2].trim();
|
||||
} else {
|
||||
const cm = TIMELINE_LINE_RE_CN.exec(lines[i]);
|
||||
if (!cm) { i++; continue; }
|
||||
// Normalize Chinese date to YYYY-MM-DD
|
||||
date = `${cm[1]}-${cm[2].padStart(2, '0')}-${cm[3].padStart(2, '0')}`;
|
||||
summary = cm[4].trim();
|
||||
}
|
||||
const date = m[1];
|
||||
const summary = m[2].trim();
|
||||
if (!isValidDate(date) || summary.length === 0) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isValidDate(date) || summary.length === 0) { i++; continue; }
|
||||
// Collect optional detail lines (indented, until next date or heading).
|
||||
const detailLines: string[] = [];
|
||||
let j = i + 1;
|
||||
@@ -1266,6 +1290,46 @@ function isValidDate(s: string): boolean {
|
||||
return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d;
|
||||
}
|
||||
|
||||
/** Input for {@link deriveTimelineAnchor}: a page's identity + its computed content date. */
|
||||
export interface TimelineAnchorInput {
|
||||
slug: string;
|
||||
title?: string | null;
|
||||
effectiveDate?: Date | string | null;
|
||||
effectiveDateSource?: EffectiveDateSource | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Anchor a single timeline entry from a page's computed content date, for pages
|
||||
* whose body carries no parseable timeline line.
|
||||
*
|
||||
* Comms- and calendar-dominated brains keep the date in frontmatter or the
|
||||
* filename (slug `2026-04-24-...`), not in the prose, so `parseTimelineEntries`
|
||||
* returns nothing and the page-level `timeline` table stays empty even though
|
||||
* the page is firmly dated — leaving `get_timeline` and the brain-score
|
||||
* `timeline_coverage` component blind to it. This recovers that signal from the
|
||||
* already-computed `effective_date` (no re-parsing). (It does NOT feed the
|
||||
* facts-based `find_trajectory`, which reads the `facts` table by entity_slug.)
|
||||
*
|
||||
* Fires ONLY for a trustworthy content date — frontmatter (`event_date` / `date`
|
||||
* / `published`) or the `filename` date — never the `fallback` source, which is
|
||||
* `updated_at` (link-churn noise, not when the thing happened). Returns null
|
||||
* when no trustworthy date is available. Callers MUST apply this only when body
|
||||
* parsing yields zero entries, so it can never shadow a real in-body timeline.
|
||||
*/
|
||||
export function deriveTimelineAnchor(input: TimelineAnchorInput): TimelineCandidate | null {
|
||||
const { slug, title, effectiveDate, effectiveDateSource } = input;
|
||||
if (!effectiveDate) return null;
|
||||
// 'fallback' === updated_at; the rest ('event_date'|'date'|'published'|'filename')
|
||||
// are real content dates. null/undefined source is not trustworthy either.
|
||||
if (effectiveDateSource == null || effectiveDateSource === 'fallback') return null;
|
||||
const dt = typeof effectiveDate === 'string' ? new Date(effectiveDate) : effectiveDate;
|
||||
if (!(dt instanceof Date) || Number.isNaN(dt.getTime())) return null;
|
||||
const iso = dt.toISOString().slice(0, 10);
|
||||
if (!isValidDate(iso)) return null;
|
||||
const summary = (title ?? '').trim() || slug.split('/').pop() || slug;
|
||||
return { date: iso, summary, detail: '' };
|
||||
}
|
||||
|
||||
// ─── Auto-link config ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
+211
-1
@@ -11,7 +11,7 @@ import type { GBrainConfig } from './config.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import { importFromContent } from './import-file.ts';
|
||||
import { writePageThrough } from './write-through.ts';
|
||||
import { hybridSearch, hybridSearchCached, stampContentFlags } from './search/hybrid.ts';
|
||||
import { hybridSearch, hybridSearchCached, stampContentFlags, stampUnverifiedExtractions } from './search/hybrid.ts';
|
||||
import { expandQuery } from './search/expansion.ts';
|
||||
import { dedupResults } from './search/dedup.ts';
|
||||
import { captureEvalCandidate, isEvalCaptureEnabled, isEvalScrubEnabled } from './eval-capture.ts';
|
||||
@@ -21,6 +21,8 @@ import { isFactsBackstopEligible } from './facts/eligibility.ts';
|
||||
import { stripTakesFence } from './takes-fence.ts';
|
||||
import { stripFactsFence } from './facts-fence.ts';
|
||||
import { getContentFlag } from './quarantine.ts';
|
||||
import { unverifiedExtractionFragment, isUnverifiedExtraction, EXTRACTION_STATUS_KEY, STATUS_VERIFIED } from './extraction-review.ts';
|
||||
import { buildVisibilityClause } from './search/sql-ranking.ts';
|
||||
import { bumpLastRetrievedAt } from './last-retrieved.ts';
|
||||
import { isSearchMode } from './search/mode.ts';
|
||||
import { stampEvidence } from './search/evidence.ts';
|
||||
@@ -1625,6 +1627,10 @@ const search: Operation = {
|
||||
// agent-warning channel (hybridSearch stamps it; this branch bypasses
|
||||
// hybridSearch, so stamp explicitly). Fail-open inside the helper.
|
||||
await stampContentFlags(ctx.engine, results);
|
||||
// #160: same for the unverified auto-extracted stub marker (no boost
|
||||
// to cancel on this path — keyword-only never applies the compiled-
|
||||
// truth boost — but the provenance marker must still surface).
|
||||
await stampUnverifiedExtractions(ctx.engine, results);
|
||||
bumpLastRetrievedAt(ctx.engine, results.map((r) => r.page_id));
|
||||
maybeCaptureSearch(ctx, queryText, results, Date.now() - startedAt, false);
|
||||
return results;
|
||||
@@ -5671,6 +5677,208 @@ const chronicle_backfill: Operation = {
|
||||
cliHints: { name: 'chronicle-backfill' },
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extraction quarantine lane (issue #160)
|
||||
//
|
||||
// `extractAndEnrich` regex-extracts entity names from arbitrary text and
|
||||
// creates people/ + companies/ stub pages. These three ops are its ONLY
|
||||
// sanctioned surface:
|
||||
// - extract_entities — run extraction. Direct authoritative writes need
|
||||
// BOTH the trusted local CLI (ctx.remote === false)
|
||||
// AND the explicit --trusted-extraction flag;
|
||||
// everything else lands in the quarantine lane
|
||||
// (frontmatter provenance/status markers).
|
||||
// - extraction_pending — list unverified stubs awaiting review.
|
||||
// - extraction_review — promote (status → verified) or reject
|
||||
// (soft-delete) in batch. Owner-only (fail-closed
|
||||
// on ctx.remote): THIS surface never lets a remote
|
||||
// caller flip the status markers. Scope note: the
|
||||
// markers are ordinary frontmatter, so a caller who
|
||||
// already holds generic remote put_page write scope
|
||||
// can rewrite the page (markers included) — that
|
||||
// caller could equally author an unmarked people/
|
||||
// page directly, so the lane adds no privilege
|
||||
// there; put_page authz is its own boundary.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Resource guards for extract_entities (#160 hardening): bound the work a
|
||||
// single remote write-scope call can trigger. ponytail: flat caps; make them
|
||||
// config knobs only if a real workload hits them.
|
||||
const MAX_EXTRACT_TEXT_CHARS = 200_000;
|
||||
const MAX_EXTRACT_ENTITIES = 200;
|
||||
|
||||
const extract_entities: Operation = {
|
||||
name: 'extract_entities',
|
||||
description: 'Extract entity names (people, companies) from text and create/update their brain stub pages. Stubs from untrusted input land in the quarantine lane (frontmatter `provenance: auto-extracted` + `status: unverified`) — excluded from authoritative retrieval boosts until reviewed. Direct authoritative writes require the trusted local CLI AND --trusted-extraction.',
|
||||
params: {
|
||||
text: { type: 'string', required: true, description: 'The text to extract entities from (email, transcript, pasted content, …). Max 200k characters — split larger inputs.' },
|
||||
source_slug: { type: 'string', required: true, description: 'Slug of the source page the text came from (used for backlinks + timeline attribution).' },
|
||||
trusted_extraction: { type: 'boolean', required: false, description: 'Local CLI only: write stubs directly as authoritative pages, skipping the quarantine lane. Ignored (always quarantined) for remote callers.' },
|
||||
},
|
||||
mutating: true,
|
||||
scope: 'write',
|
||||
handler: async (ctx, p) => {
|
||||
// Trust rule (#160, fail-closed like the CV6 provenance gate above):
|
||||
// `ctx.remote === false` is the ONLY truthy condition that can admit a
|
||||
// direct authoritative write, and even then the caller must opt in
|
||||
// explicitly. Remote/unset trust → quarantine lane, flag ignored.
|
||||
const trusted = ctx.remote === false && p.trusted_extraction === true;
|
||||
const text = p.text as string;
|
||||
// Resource guards: the greedy name regex on a huge paste can yield tens
|
||||
// of thousands of "entities", each costing several DB round-trips. Cap
|
||||
// input size loudly and entity count softly (surfaced as `truncated`).
|
||||
if (text.length > MAX_EXTRACT_TEXT_CHARS) {
|
||||
throw new OperationError(
|
||||
'invalid_params',
|
||||
`extract_entities: text is ${text.length} chars (max ${MAX_EXTRACT_TEXT_CHARS}).`,
|
||||
'Split the input and call extract_entities per section.',
|
||||
);
|
||||
}
|
||||
if (ctx.dryRun) return { dry_run: true, action: 'extract_entities', trusted };
|
||||
const { extractEntities, enrichEntities } = await import('./enrichment-service.ts');
|
||||
const found = extractEntities(text);
|
||||
const capped = found.slice(0, MAX_EXTRACT_ENTITIES);
|
||||
const results = await enrichEntities(
|
||||
ctx.engine,
|
||||
capped.map((e) => ({ entityName: e.name, entityType: e.type, context: e.context, sourceSlug: p.source_slug as string })),
|
||||
{
|
||||
trusted,
|
||||
...(ctx.sourceId ? { sourceId: ctx.sourceId } : {}),
|
||||
// Pure local DB writes — no external API call to pace, so the
|
||||
// system-load capacity gate would only stall the caller.
|
||||
throttle: false,
|
||||
},
|
||||
);
|
||||
return {
|
||||
status: 'ok',
|
||||
trusted,
|
||||
quarantined: results.filter((r) => r.quarantined === true).length,
|
||||
count: results.length,
|
||||
entities_found: found.length,
|
||||
truncated: found.length > capped.length,
|
||||
entities: results,
|
||||
};
|
||||
},
|
||||
cliHints: { name: 'extract-entities' },
|
||||
};
|
||||
|
||||
const extraction_pending: Operation = {
|
||||
name: 'extraction_pending',
|
||||
description: 'List unverified auto-extracted entity stubs awaiting owner review (the quarantine lane from extract_entities). Promote or reject them with extraction_review.',
|
||||
params: {
|
||||
limit: { type: 'number', required: false, description: 'Max rows (default 100, cap 500).' },
|
||||
offset: { type: 'number', required: false, description: 'Pagination offset.' },
|
||||
},
|
||||
scope: 'read',
|
||||
handler: async (ctx, p) => {
|
||||
const limit = Math.min(Math.max(Number(p.limit ?? 100) || 100, 1), 500);
|
||||
const offset = Math.max(Number(p.offset ?? 0) || 0, 0);
|
||||
// Read-side source isolation: route through sourceScopeOpts (federated
|
||||
// array > scalar > nothing), applied in SQL below.
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
const params: unknown[] = [];
|
||||
let srcClause = '';
|
||||
if (scope.sourceIds && scope.sourceIds.length > 0) {
|
||||
params.push(scope.sourceIds);
|
||||
srcClause = `AND p.source_id = ANY($${params.length}::text[])`;
|
||||
} else if (scope.sourceId) {
|
||||
params.push(scope.sourceId);
|
||||
srcClause = `AND p.source_id = $${params.length}`;
|
||||
}
|
||||
params.push(limit, offset);
|
||||
const rows = await ctx.engine.executeRaw<{
|
||||
slug: string; title: string; type: string; source_id: string;
|
||||
extracted_from: string | null; created_at: string;
|
||||
}>(
|
||||
`SELECT p.slug, p.title, p.type, p.source_id,
|
||||
p.frontmatter ->> 'source' AS extracted_from,
|
||||
p.created_at::text AS created_at
|
||||
FROM pages p
|
||||
JOIN sources s ON s.id = p.source_id
|
||||
WHERE ${unverifiedExtractionFragment('p')}
|
||||
${buildVisibilityClause('p', 's')}
|
||||
${srcClause}
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT $${params.length - 1} OFFSET $${params.length}`,
|
||||
params,
|
||||
);
|
||||
return { count: rows.length, pending: rows };
|
||||
},
|
||||
cliHints: { name: 'extraction-pending' },
|
||||
};
|
||||
|
||||
const extraction_review: Operation = {
|
||||
name: 'extraction_review',
|
||||
description: 'Promote or reject unverified auto-extracted entity stubs (batch). Promote flips `status` to verified (provenance kept for audit); reject soft-deletes the stub. Owner-only: this op is refused for any non-local caller. (The markers are ordinary frontmatter — the boundary against rewriting them wholesale is put_page write authz, same as for any page.)',
|
||||
params: {
|
||||
action: { type: 'string', required: true, description: "'promote' or 'reject'." },
|
||||
slugs: { type: 'array', required: true, items: { type: 'string' }, description: 'Stub slugs to act on (batch).' },
|
||||
},
|
||||
mutating: true,
|
||||
scope: 'write',
|
||||
localOnly: true,
|
||||
handler: async (ctx, p) => {
|
||||
// The review decision IS the trust gate — if a remote caller could
|
||||
// promote, injected content could self-promote and the quarantine lane
|
||||
// would be decorative. Fail-closed: only strictly-local callers pass.
|
||||
if (ctx.remote !== false) {
|
||||
throw new OperationError(
|
||||
'permission_denied',
|
||||
'extraction_review is owner-only: promote/reject decisions must come from the trusted local CLI.',
|
||||
'Run `gbrain extraction-review <promote|reject> --slugs ...` on the host machine.',
|
||||
);
|
||||
}
|
||||
const action = p.action as string;
|
||||
if (action !== 'promote' && action !== 'reject') {
|
||||
throw new OperationError('invalid_params', `extraction_review: action must be 'promote' or 'reject'; got '${action}'.`);
|
||||
}
|
||||
// CLI passes `--slugs a,b,c` as one string; MCP passes a real array.
|
||||
const slugs = Array.isArray(p.slugs)
|
||||
? (p.slugs as string[])
|
||||
: typeof p.slugs === 'string'
|
||||
? p.slugs.split(',').map((s) => s.trim()).filter(Boolean)
|
||||
: [];
|
||||
if (slugs.length === 0) {
|
||||
throw new OperationError('invalid_params', 'extraction_review: slugs must be a non-empty array (CLI: --slugs slug1,slug2).');
|
||||
}
|
||||
if (ctx.dryRun) return { dry_run: true, action: `extraction_review:${action}`, slugs };
|
||||
const results: Array<{ slug: string; status: string }> = [];
|
||||
for (const slug of slugs) {
|
||||
const page = await ctx.engine.getPage(slug, ctx.sourceId ? { sourceId: ctx.sourceId } : undefined);
|
||||
if (!page) {
|
||||
results.push({ slug, status: 'not_found' });
|
||||
continue;
|
||||
}
|
||||
if (!isUnverifiedExtraction(page.frontmatter)) {
|
||||
results.push({ slug, status: 'not_unverified' });
|
||||
continue;
|
||||
}
|
||||
if (action === 'promote') {
|
||||
// Frontmatter-only flip via a targeted JSONB merge — NOT putPage,
|
||||
// whose upsert would reset non-carried columns (page_kind →
|
||||
// 'markdown', content_hash, …) for a change that only touches one
|
||||
// frontmatter key. provenance stays 'auto-extracted' as the audit
|
||||
// trail of HOW the page came to exist; status → 'verified' records
|
||||
// the owner's call. jsonb_build_object binds as text (no
|
||||
// JSON.stringify-into-::jsonb hazard); identical on both engines.
|
||||
await ctx.engine.executeRaw(
|
||||
`UPDATE pages
|
||||
SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || jsonb_build_object($1::text, $2::text),
|
||||
updated_at = now()
|
||||
WHERE slug = $3 AND source_id = $4`,
|
||||
[EXTRACTION_STATUS_KEY, STATUS_VERIFIED, slug, page.source_id],
|
||||
);
|
||||
results.push({ slug, status: 'promoted' });
|
||||
} else {
|
||||
await ctx.engine.softDeletePage(slug, { sourceId: page.source_id });
|
||||
results.push({ slug, status: 'rejected' });
|
||||
}
|
||||
}
|
||||
return { status: 'ok', action, results };
|
||||
},
|
||||
cliHints: { name: 'extraction-review', positional: ['action'] },
|
||||
};
|
||||
|
||||
export const operations: Operation[] = [
|
||||
// Page CRUD
|
||||
get_page, put_page, delete_page, list_pages,
|
||||
@@ -5727,6 +5935,8 @@ export const operations: Operation[] = [
|
||||
volunteer_chronicle, chronicle_backfill,
|
||||
// v0.43 (#2095): push-based context
|
||||
volunteer_context,
|
||||
// Extraction quarantine lane (#160): gated entity extraction + review queue
|
||||
extract_entities, extraction_pending, extraction_review,
|
||||
// v0.31: hot memory (facts table)
|
||||
extract_facts, recall, forget_fact,
|
||||
// v0.32.6: contradiction probe MCP surface (M3)
|
||||
|
||||
@@ -58,6 +58,7 @@ import { finalizeLastSeen } from './chronicle/last-seen.ts';
|
||||
import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from './extraction-review.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
import {
|
||||
@@ -2072,7 +2073,10 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Built on the bare `slug` output column: applied inside the `scored` CTE
|
||||
// whose FROM is the single relation `hnsw_candidates`, so unqualified
|
||||
// `slug` resolves cleanly (T1 per-page pool restructure).
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail);
|
||||
// issue #160: guard predicate projected as `unverified_stub` in
|
||||
// hnsw_candidates (parity with postgres-engine) so unverified stubs get
|
||||
// factor 1.0, not the people/ 1.2x, inside the pre-LIMIT re-rank.
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail, 'unverified_stub');
|
||||
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
|
||||
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
|
||||
const innerLimit = offset + Math.max(limit * 5, 100);
|
||||
@@ -2148,6 +2152,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL
|
||||
THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject,
|
||||
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
|
||||
(${unverifiedExtractionFragment('p')}) AS unverified_stub,
|
||||
1 - (cc.${col} <=> ${castSql}) AS raw_score
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
@@ -3418,6 +3423,20 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return result;
|
||||
}
|
||||
|
||||
async getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>> {
|
||||
if (pageIds.length === 0) return new Set();
|
||||
// Parity with PostgresEngine.getUnverifiedExtractionPageIds (issue #160).
|
||||
// Predicate is the shared unverifiedExtractionFragment so this query and
|
||||
// the SQL-side source-boost guard can never drift.
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id FROM pages
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ${unverifiedExtractionFragment('pages')}`,
|
||||
[pageIds]
|
||||
);
|
||||
return new Set((rows as { id: number }[]).map((r) => Number(r.id)));
|
||||
}
|
||||
|
||||
async getPageTimestamps(slugs: string[]): Promise<Map<string, Date>> {
|
||||
if (slugs.length === 0) return new Map();
|
||||
const { rows } = await this.db.query(
|
||||
|
||||
@@ -65,6 +65,7 @@ import { logConnectionEvent } from './connection-audit.ts';
|
||||
import { validateSlug, contentHash, rowToPage, rowToStalePage, rowToChunk, rowToSearchResult, parseEmbedding, tryParseEmbedding, takeRowToTake, takeHitRowToHit, isUndefinedTableError, warnOncePerProcess } from './utils.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from './extraction-review.ts';
|
||||
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts';
|
||||
import { DELETE_BATCH_SIZE } from './engine-constants.ts';
|
||||
import { SOURCE_CONFIG_OBJECT_SQL } from './source-config-sql.ts';
|
||||
@@ -2120,7 +2121,10 @@ export class PostgresEngine implements BrainEngine {
|
||||
// innerLimit scales with offset to preserve the pagination contract:
|
||||
// a fixed cap of 100 would silently empty offset > 100.
|
||||
const boostMap = resolveBoostMap();
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail);
|
||||
// issue #160: the guard predicate is projected as `unverified_stub` in
|
||||
// hnsw_candidates (frontmatter isn't otherwise available at re-rank), so
|
||||
// unverified auto-extracted stubs get factor 1.0, not the people/ 1.2x.
|
||||
const sourceFactorCaseOnSlug = buildSourceFactorCase('slug', boostMap, opts?.detail, 'unverified_stub');
|
||||
const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes);
|
||||
const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes);
|
||||
const innerLimit = offset + Math.max(limit * 5, 100);
|
||||
@@ -2220,6 +2224,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL
|
||||
THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject,
|
||||
cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source,
|
||||
(${unverifiedExtractionFragment('p')}) AS unverified_stub,
|
||||
1 - (cc.${col} <=> ${castSql}) AS raw_score
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
@@ -3571,6 +3576,20 @@ export class PostgresEngine implements BrainEngine {
|
||||
return result;
|
||||
}
|
||||
|
||||
async getUnverifiedExtractionPageIds(pageIds: number[]): Promise<Set<number>> {
|
||||
if (pageIds.length === 0) return new Set();
|
||||
const sql = this.sql;
|
||||
// Predicate is the shared unverifiedExtractionFragment (issue #160) so
|
||||
// this query and the SQL-side source-boost guard can never drift.
|
||||
const rows = await sql.unsafe(
|
||||
`SELECT id FROM pages
|
||||
WHERE id = ANY($1::int[])
|
||||
AND ${unverifiedExtractionFragment('pages')}`,
|
||||
[pageIds] as never,
|
||||
);
|
||||
return new Set((rows as unknown as { id: number }[]).map((r) => Number(r.id)));
|
||||
}
|
||||
|
||||
async getPageTimestamps(slugs: string[]): Promise<Map<string, Date>> {
|
||||
if (slugs.length === 0) return new Map();
|
||||
const sql = this.sql;
|
||||
|
||||
@@ -76,6 +76,37 @@ export async function stampContentFlags(engine: BrainEngine, results: SearchResu
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160). Stamps `SearchResult.unverified`
|
||||
* for any result whose page is an unverified auto-extracted entity stub
|
||||
* (frontmatter `provenance: 'auto-extracted'` + `status: 'unverified'`).
|
||||
* MUST run PRE-fusion: rrfFusion/rrfFusionWeighted read the flag to skip the
|
||||
* COMPILED_TRUTH_BOOST for these pages, so a stub fabricated by hostile
|
||||
* ingested text ranks as ordinary content, never with entity authority.
|
||||
* One batched query over the candidate arms' page_ids. Fail-open on the
|
||||
* fetch (a marker-fetch failure must not break retrieval) — the boost then
|
||||
* applies, but the SQL-side source-boost guard still holds.
|
||||
*/
|
||||
export async function stampUnverifiedExtractions(
|
||||
engine: BrainEngine,
|
||||
results: SearchResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
try {
|
||||
const ids = [...new Set(
|
||||
results.map((r) => r.page_id).filter((n): n is number => typeof n === 'number' && Number.isFinite(n)),
|
||||
)];
|
||||
if (ids.length === 0) return;
|
||||
const unverified = await engine.getUnverifiedExtractionPageIds(ids);
|
||||
if (unverified.size === 0) return;
|
||||
for (const r of results) {
|
||||
if (unverified.has(r.page_id)) r.unverified = true;
|
||||
}
|
||||
} catch {
|
||||
// best-effort: never break retrieval.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.42.20.0 — bounded drain (was an unbounded `Promise.allSettled`, codex
|
||||
* confirmed; TODOS retrofit). Mirrors `awaitPendingLastRetrievedWrites`: races
|
||||
@@ -1101,12 +1132,37 @@ export async function hybridSearch(
|
||||
// provider (Voyage, ZE) works fine.
|
||||
const { isAvailable } = await import('../ai/gateway.ts');
|
||||
const providerProbe = resolvedCol.embeddingModel || undefined;
|
||||
if (!isAvailable('embedding', providerProbe)) {
|
||||
// Image/both/unified routing embeds via the MULTIMODAL provider, not the
|
||||
// text provider — so a multimodal-only install (text provider absent) must
|
||||
// still reach the multimodal branch below. Probe the multimodal provider
|
||||
// explicitly and only short-circuit when neither the text provider nor (for
|
||||
// multimodal-routed queries) the multimodal provider is reachable. Without
|
||||
// this guard a multimodal-only install would fall to keyword-only here and
|
||||
// never run the image/unified vector path.
|
||||
const multimodalProviderProbe =
|
||||
cfgForColumn?.embedding_multimodal_model ?? 'voyage:voyage-multimodal-3';
|
||||
// The LLM intent tie-break (below) can escalate a regex-'text' query to
|
||||
// 'image'/'both'; account for that possibility so an ambiguous query on a
|
||||
// multimodal-only install still reaches the multimodal branch.
|
||||
const mayEscalateToMultimodal =
|
||||
earlyModality === 'text' &&
|
||||
resolvedMode.cross_modal_llm_intent &&
|
||||
isAmbiguousModalityQuery(query);
|
||||
const willTryMultimodal =
|
||||
(resolvedMode.unified_multimodal === true ||
|
||||
earlyModality === 'image' ||
|
||||
earlyModality === 'both' ||
|
||||
mayEscalateToMultimodal) &&
|
||||
isAvailable('embedding', multimodalProviderProbe);
|
||||
if (!isAvailable('embedding', providerProbe) && !willTryMultimodal) {
|
||||
// v0.43 — fuse the relational arm with keyword so typed-edge answers
|
||||
// survive on the no-embedding-provider path (the relational win is most
|
||||
// valuable exactly when vector is unavailable). The title arm fuses here
|
||||
// too — an exact-title lookup on a keyless install is precisely where
|
||||
// chunk-grain keyword FTS alone fails (D1).
|
||||
// issue #160: stamp unverified stubs BEFORE fusion so the compiled-truth
|
||||
// boost skips them (flag survives fusion's result spread).
|
||||
await stampUnverifiedExtractions(engine, [...keywordResults, ...titleResults, ...relationalList]);
|
||||
let noEmbedResults = keywordResults;
|
||||
if (relationalList.length > 0 || titleResults.length > 0) {
|
||||
const fk = opts?.rrfK ?? RRF_K;
|
||||
@@ -1233,7 +1289,10 @@ export async function hybridSearch(
|
||||
if (unifiedRouting) {
|
||||
try {
|
||||
const { isAvailable: aiIsAvailable, embedQueryMultimodal } = await import('../ai/gateway.ts');
|
||||
if (!aiIsAvailable('embedding')) {
|
||||
// Probe the MULTIMODAL provider, not the global default — on a
|
||||
// multimodal-only install the global default (text) is absent but the
|
||||
// multimodal provider is configured, and unified routing embeds via it.
|
||||
if (!aiIsAvailable('embedding', multimodalProviderProbe)) {
|
||||
throw new Error('gateway not configured for embedding — unified multimodal would also fail');
|
||||
}
|
||||
const unifiedEmbedding = await embedQueryMultimodal(query);
|
||||
@@ -1268,7 +1327,10 @@ export async function hybridSearch(
|
||||
// OR the embed throws, log a structured warning and fall through to text.
|
||||
try {
|
||||
const { isAvailable: aiIsAvailable, embedQueryMultimodal } = await import('../ai/gateway.ts');
|
||||
if (!aiIsAvailable('embedding')) {
|
||||
// Probe the MULTIMODAL provider, not the global default — the image side
|
||||
// embeds via the multimodal model, which may be configured even when the
|
||||
// text/global-default embedding provider is absent (multimodal-only).
|
||||
if (!aiIsAvailable('embedding', multimodalProviderProbe)) {
|
||||
throw new Error('gateway not configured for embedding — multimodal would also fail');
|
||||
}
|
||||
const imageEmbedding = await embedQueryMultimodal(query);
|
||||
@@ -1342,6 +1404,9 @@ export async function hybridSearch(
|
||||
// v0.43: fuse the relational arm with keyword via RRF so typed-edge
|
||||
// answers survive even when vector is unavailable. The title arm fuses
|
||||
// here too (same rationale as the no-embedding-provider path — D1).
|
||||
// issue #160: stamp unverified stubs BEFORE fusion (see the
|
||||
// no-embedding-provider path for rationale).
|
||||
await stampUnverifiedExtractions(engine, [...keywordResults, ...titleResults, ...relationalList]);
|
||||
let fallbackResults = keywordResults;
|
||||
if (relationalList.length > 0 || titleResults.length > 0) {
|
||||
const fk = opts?.rrfK ?? RRF_K;
|
||||
@@ -1431,6 +1496,10 @@ export async function hybridSearch(
|
||||
allLists.push({ list: relationalList, k: baseRrfK });
|
||||
}
|
||||
|
||||
// issue #160: stamp unverified auto-extracted stubs across ALL candidate
|
||||
// arms BEFORE fusion so the compiled-truth authority boost skips them.
|
||||
await stampUnverifiedExtractions(engine, allLists.flatMap((l) => l.list));
|
||||
|
||||
let fused = rrfFusionWeighted(allLists, detail !== 'high');
|
||||
|
||||
// Cosine re-scoring before dedup so semantically better chunks survive.
|
||||
@@ -1997,7 +2066,9 @@ export function rrfFusionWeighted(
|
||||
if (maxScore > 0) {
|
||||
for (const e of entries) {
|
||||
e.score = e.score / maxScore;
|
||||
const boost = applyBoost && e.result.chunk_source === 'compiled_truth' ? COMPILED_TRUTH_BOOST : 1.0;
|
||||
// issue #160: unverified auto-extracted stubs (stamped pre-fusion by
|
||||
// stampUnverifiedExtractions) never get the compiled-truth authority boost.
|
||||
const boost = applyBoost && e.result.chunk_source === 'compiled_truth' && e.result.unverified !== true ? COMPILED_TRUTH_BOOST : 1.0;
|
||||
e.score *= boost;
|
||||
}
|
||||
}
|
||||
@@ -2040,8 +2111,9 @@ export function rrfFusion(lists: SearchResult[][], k: number, applyBoost = true)
|
||||
const rawScore = e.score;
|
||||
e.score = e.score / maxScore;
|
||||
|
||||
// Apply compiled truth boost after normalization (skip for detail=high)
|
||||
const boost = applyBoost && e.result.chunk_source === 'compiled_truth' ? COMPILED_TRUTH_BOOST : 1.0;
|
||||
// Apply compiled truth boost after normalization (skip for detail=high;
|
||||
// skip for unverified auto-extracted stubs — issue #160)
|
||||
const boost = applyBoost && e.result.chunk_source === 'compiled_truth' && e.result.unverified !== true ? COMPILED_TRUTH_BOOST : 1.0;
|
||||
e.score *= boost;
|
||||
|
||||
if (DEBUG) {
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { quarantineFilterFragment } from '../quarantine.ts';
|
||||
import { unverifiedExtractionFragment } from '../extraction-review.ts';
|
||||
|
||||
/**
|
||||
* Escape `%`, `_`, and `\` so a string can be used as a LIKE prefix literal.
|
||||
@@ -63,6 +64,7 @@ export function buildSourceFactorCase(
|
||||
slugColumn: string,
|
||||
boostMap: Record<string, number>,
|
||||
detail: 'low' | 'medium' | 'high' | undefined,
|
||||
unverifiedGuardColumn?: string,
|
||||
): string {
|
||||
// Loose-string guard: agents passing `"HIGH"` or `"high "` over MCP/JSON
|
||||
// should still hit the temporal-bypass path. TypeScript narrows `detail`
|
||||
@@ -80,7 +82,26 @@ export function buildSourceFactorCase(
|
||||
`WHEN ${slugColumn} LIKE ${buildLikePrefixLiteral(prefix)} THEN ${factor}`
|
||||
).join(' ');
|
||||
|
||||
return `(CASE ${whens} ELSE 1.0 END)`;
|
||||
// Extraction quarantine lane (issue #160): unverified auto-extracted stubs
|
||||
// never receive the namespace-authority factor (people/ / companies/ 1.2x)
|
||||
// — they rank as ordinary content until promoted. Two forms:
|
||||
// - table-qualified slug column ('p.slug'): reference the sibling
|
||||
// `frontmatter` column inline via unverifiedExtractionFragment.
|
||||
// - bare column + `unverifiedGuardColumn`: the vector arm's re-rank CTE
|
||||
// has no frontmatter column, so its inner hnsw_candidates CTE projects
|
||||
// the predicate as a boolean (`... AS unverified_stub`) and passes the
|
||||
// column name here. Without this the 1.2x would apply INSIDE the
|
||||
// scored/best_per_page pipeline pre-LIMIT — an unverified stub could
|
||||
// outrank AND evict a legitimate page from the candidate pool, which
|
||||
// nothing downstream can restore.
|
||||
const alias = slugColumn.includes('.') ? slugColumn.split('.')[0] : null;
|
||||
const unverifiedGuard = unverifiedGuardColumn
|
||||
? `WHEN ${unverifiedGuardColumn} THEN 1.0 `
|
||||
: alias
|
||||
? `WHEN ${unverifiedExtractionFragment(alias)} THEN 1.0 `
|
||||
: '';
|
||||
|
||||
return `(CASE ${unverifiedGuard}${whens} ELSE 1.0 END)`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -699,6 +699,17 @@ export interface SearchResult {
|
||||
* Absent when the page is clean.
|
||||
*/
|
||||
content_flag?: { reason: string; detail: string };
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160): true when the result's page is
|
||||
* an unverified auto-extracted entity stub (frontmatter
|
||||
* `provenance: 'auto-extracted'` + `status: 'unverified'`). Such pages are
|
||||
* excluded from the compiled-truth authority boost and the namespace
|
||||
* source-boost — they rank as ordinary content — and this marker tells the
|
||||
* agent the page has NOT been reviewed by the owner. Stamped pre-fusion by
|
||||
* `stampUnverifiedExtractions` (hybrid.ts). Absent for reviewed/ordinary
|
||||
* pages.
|
||||
*/
|
||||
unverified?: boolean;
|
||||
/**
|
||||
* v0.36 (cross-modal wave): the chunk's modality discriminator from
|
||||
* content_chunks.modality. 'text' for the existing text-embedding rows,
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test';
|
||||
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
|
||||
import { capBatchItems, configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
|
||||
import { listRecipes, getRecipe } from '../../src/core/ai/recipes/index.ts';
|
||||
|
||||
describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warning', () => {
|
||||
@@ -49,6 +49,19 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
|
||||
expect(r!.touchpoints.embedding?.no_batch_cap).toBeUndefined();
|
||||
});
|
||||
|
||||
test('dashscope declares the documented 10-item embedding cap (max_batch_items: 10)', () => {
|
||||
// DashScope's OpenAI-compat /embeddings endpoint rejects >10-item batches
|
||||
// (documented Model Studio cap; concept from community PRs #2643/#2405).
|
||||
// max_batch_tokens stays as the aggregate token-size guard.
|
||||
const r = getRecipe('dashscope');
|
||||
expect(r, 'dashscope not registered').toBeDefined();
|
||||
expect(r!.touchpoints.embedding?.max_batch_items).toBe(10);
|
||||
expect(r!.touchpoints.embedding?.max_batch_tokens).toBe(8192);
|
||||
// 25 items pre-split into DashScope-sized groups of at most 10.
|
||||
const texts = Array.from({ length: 25 }, (_, i) => `t${i}`);
|
||||
expect(capBatchItems(texts, 10).map(b => b.length)).toEqual([10, 10, 5]);
|
||||
});
|
||||
|
||||
test('configureGateway does NOT warn for ollama/litellm/llama-server', () => {
|
||||
warnSpy.mockClear();
|
||||
resetGateway();
|
||||
|
||||
+159
-4
@@ -58,12 +58,23 @@ beforeEach(async () => {
|
||||
// Tiny gazetteer builder for pure-fn cases that don't need engine.
|
||||
function gazetteerFromEntries(entries: Omit<GazetteerEntry, 'tokens'>[]): Gazetteer {
|
||||
const TOKEN_RE = /[a-zA-Z0-9]+/g;
|
||||
const isCJK = (s: string): boolean => {
|
||||
const cp = s.codePointAt(0) ?? 0;
|
||||
return (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af);
|
||||
};
|
||||
const hasCJKTitle = (s: string): boolean => [...s].some(isCJK);
|
||||
const tokenize = (s: string): string[] => {
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase());
|
||||
return out;
|
||||
if (!hasCJKTitle(s)) {
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase());
|
||||
return out;
|
||||
}
|
||||
// CJK: split into individual characters, lowercased.
|
||||
return [...s].map(c => isCJK(c) ? c.toLowerCase() : '').filter(Boolean);
|
||||
};
|
||||
const g: Gazetteer = new Map();
|
||||
for (const raw of entries) {
|
||||
@@ -259,6 +270,128 @@ describe('findMentionedEntities — pure cases', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// CJK — entity extraction tests
|
||||
// ============================================================
|
||||
|
||||
describe('findMentionedEntities — CJK cases', () => {
|
||||
test('CJK single-name match — "纳瓦尔" in body → matched', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('我最近读了纳瓦尔的书。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(1);
|
||||
expect(mentions[0]!.slug).toBe('people/naval');
|
||||
expect(mentions[0]!.name).toBe('纳瓦尔');
|
||||
});
|
||||
|
||||
test('CJK multi-name — two different CJK entities in one body', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔和双雪涛都是作家。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(2);
|
||||
const slugs = mentions.map(m => m.slug);
|
||||
expect(slugs).toContain('people/naval');
|
||||
expect(slugs).toContain('people/shuang-xuetao');
|
||||
});
|
||||
|
||||
test('CJK first-mention-only — repeated name → single link', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔说过。然后纳瓦尔又说过。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('CJK self-link guard — entity page mentioning itself is skipped', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔是一位投资人。', g, {
|
||||
fromSlug: 'people/naval', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK cross-source guard — entity in different source skipped', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'team-b', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔写了这本书。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'team-a',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK code-block stripping — CJK name inside ``` is skipped, outside matched', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
// "纳瓦尔" only appears inside code block → should be skipped.
|
||||
const body = '```\n纳瓦尔\n```\n只有代码块里面有。';
|
||||
const mentions = findMentionedEntities(body, g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('CJK determinism — same output across 10 calls', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||||
]);
|
||||
const body = '纳瓦尔和双雪涛。纳瓦尔再说一次。';
|
||||
const refs = new Set<string>();
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const mentions = findMentionedEntities(body, g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
refs.add(JSON.stringify(mentions));
|
||||
}
|
||||
expect(refs.size).toBe(1);
|
||||
});
|
||||
|
||||
test('CJK mixed body — CJK entity matched in body with ASCII around it', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('Acme was founded by 纳瓦尔 in 2020.', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(2);
|
||||
const slugs = mentions.map(m => m.slug);
|
||||
expect(slugs).toContain('people/naval');
|
||||
expect(slugs).toContain('companies/acme');
|
||||
});
|
||||
|
||||
test('CJK empty gazetteer — no false positives', () => {
|
||||
const g: Gazetteer = new Map();
|
||||
const mentions = findMentionedEntities('纳瓦尔和双雪涛。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK empty text → empty result', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// buildGazetteer — engine-backed tests
|
||||
// ============================================================
|
||||
@@ -366,4 +499,26 @@ describe('buildGazetteer — engine integration', () => {
|
||||
// forces a deliberate change (and a corresponding test update).
|
||||
expect(LINKABLE_ENTITY_TYPES).toEqual(['person', 'company', 'organization', 'entity']);
|
||||
});
|
||||
|
||||
// CJK — engine-backed tests
|
||||
test('CJK entity with 2-char title enters gazetteer with char-level tokens', async () => {
|
||||
await engine.putPage('people/naval', {
|
||||
type: 'person', title: '纳瓦尔', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||||
});
|
||||
const g = await buildGazetteer(engine);
|
||||
// "纳瓦尔" tokenized as ["纳","瓦","尔"] → key is "纳"
|
||||
expect(g.has('纳')).toBe(true);
|
||||
const bucket = g.get('纳')!;
|
||||
expect(bucket.length).toBe(1);
|
||||
expect(bucket[0]!.tokens).toEqual(['纳', '瓦', '尔']);
|
||||
expect(bucket[0]!.slug).toBe('people/naval');
|
||||
});
|
||||
|
||||
test('CJK single-char title (cjkCharCount < 2) excluded from gazetteer', async () => {
|
||||
await engine.putPage('people/x', {
|
||||
type: 'person', title: '谢', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||||
});
|
||||
const g = await buildGazetteer(engine);
|
||||
expect(g.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
normalizeBlockConversation,
|
||||
looksLikeBlockConversation,
|
||||
} from '../src/core/conversation-parser/normalize-block.ts';
|
||||
import { parseConversation } from '../src/core/conversation-parser/parse.ts';
|
||||
|
||||
// A realistic Slack-collector page body (the format gbrain's own collector emits).
|
||||
const SLACK_DM = `# DM (group) with Hugh, Karyshma, Theo — 2026-06-15
|
||||
|
||||
- **Theo** (Mon 11:18)
|
||||
Hey everyone — quick note on the *real fiscal value* we surface after accounting.
|
||||
|
||||
It's a huge win at renewal and dents churn.
|
||||
- **Juan** (Mon 11:20)
|
||||
Agreed. Let's make sure we capture it for all ongoing customers.`;
|
||||
|
||||
describe('looksLikeBlockConversation', () => {
|
||||
test('detects the block header signature', () => {
|
||||
expect(looksLikeBlockConversation(SLACK_DM)).toBe(true);
|
||||
expect(looksLikeBlockConversation('- **Theo** (16:36)\n body')).toBe(true);
|
||||
expect(looksLikeBlockConversation('- **Theo** (11:18 AM)\n body')).toBe(true);
|
||||
});
|
||||
|
||||
test('is false for canonical single-line content (no false trigger)', () => {
|
||||
expect(looksLikeBlockConversation('**Theo** (11:18): hi there')).toBe(false);
|
||||
expect(looksLikeBlockConversation('**Theo** (2026-06-15 11:18): hi')).toBe(false);
|
||||
expect(looksLikeBlockConversation('just some prose with no chat at all')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeBlockConversation', () => {
|
||||
test('collapses header + indented multi-paragraph body to one canonical line', () => {
|
||||
const out = normalizeBlockConversation(SLACK_DM).split('\n');
|
||||
expect(out).toEqual([
|
||||
"**Theo** (11:18): Hey everyone — quick note on the *real fiscal value* we surface after accounting. It's a huge win at renewal and dents churn.",
|
||||
"**Juan** (11:20): Agreed. Let's make sure we capture it for all ongoing customers.",
|
||||
]);
|
||||
});
|
||||
|
||||
test('drops the page-title line and leading blanks', () => {
|
||||
const out = normalizeBlockConversation(SLACK_DM);
|
||||
expect(out.startsWith('# DM')).toBe(false);
|
||||
expect(out.startsWith('**Theo**')).toBe(true);
|
||||
});
|
||||
|
||||
test('converts 12h am/pm to 24h', () => {
|
||||
expect(normalizeBlockConversation('- **A** (1:05 PM)\n hi')).toBe('**A** (13:05): hi');
|
||||
expect(normalizeBlockConversation('- **A** (12:00 AM)\n midnight')).toBe('**A** (00:00): midnight');
|
||||
expect(normalizeBlockConversation('- **A** (12:30 PM)\n noon-ish')).toBe('**A** (12:30): noon-ish');
|
||||
});
|
||||
|
||||
test('keeps day-of-week out of the emitted time', () => {
|
||||
expect(normalizeBlockConversation('- **A** (Tue 09:07)\n morning')).toBe('**A** (09:07): morning');
|
||||
});
|
||||
|
||||
test('is a strict no-op on canonical single-line content', () => {
|
||||
const canonical = '**Theo** (11:18): hi\n**Juan** (11:20): yo';
|
||||
expect(normalizeBlockConversation(canonical)).toBe(canonical);
|
||||
});
|
||||
|
||||
test('a message with no body emits an empty-body line', () => {
|
||||
expect(normalizeBlockConversation('- **A** (10:00)')).toBe('**A** (10:00): ');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseConversation integration — block format now yields messages', () => {
|
||||
test('Slack-collector body parses to 2 messages via the normalize pre-pass', () => {
|
||||
const res = parseConversation(SLACK_DM, { fallbackDate: '2026-06-15' });
|
||||
expect(res.messages.length).toBe(2);
|
||||
expect(res.messages[0].speaker).toBe('Theo');
|
||||
expect(res.messages[1].speaker).toBe('Juan');
|
||||
expect(res.phase).not.toBe('no_match');
|
||||
});
|
||||
|
||||
test('canonical content still parses unchanged (no regression)', () => {
|
||||
const res = parseConversation('**Theo** (11:18): hi there', { fallbackDate: '2026-06-15' });
|
||||
expect(res.messages.length).toBe(1);
|
||||
expect(res.messages[0].speaker).toBe('Theo');
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,7 @@ import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
isSourceUnchangedSinceSync,
|
||||
probeSourceGitState,
|
||||
_setGitHeadProbeForTests,
|
||||
_setGitCleanProbeForTests,
|
||||
type GitHeadProbe,
|
||||
@@ -176,3 +177,60 @@ describe('isSourceUnchangedSinceSync — requireCleanWorkingTree (D7)', () => {
|
||||
expect(cleanCalls).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('probeSourceGitState — three-state verdict', () => {
|
||||
test('state 1: HEAD matches + clean → unchanged', () => {
|
||||
_setGitHeadProbeForTests(() => 'abc123');
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
expect(probeSourceGitState('/tmp/repo', 'abc123', { requireCleanWorkingTree: 'ignore-untracked' }))
|
||||
.toBe('unchanged');
|
||||
});
|
||||
|
||||
test('state 2: HEAD probe null (clone missing / not a repo / git error) → unavailable', () => {
|
||||
_setGitHeadProbeForTests(() => null);
|
||||
expect(probeSourceGitState('/tmp/gone', 'abc123')).toBe('unavailable');
|
||||
});
|
||||
|
||||
test('state 3: HEAD mismatch → changed', () => {
|
||||
_setGitHeadProbeForTests(() => 'def456');
|
||||
expect(probeSourceGitState('/tmp/repo', 'abc123')).toBe('changed');
|
||||
});
|
||||
|
||||
test('state 4: dirty tree with readable HEAD → changed (NOT unavailable)', () => {
|
||||
_setGitHeadProbeForTests(() => 'abc123');
|
||||
_setGitCleanProbeForTests(() => false);
|
||||
expect(probeSourceGitState('/tmp/repo', 'abc123', { requireCleanWorkingTree: true }))
|
||||
.toBe('changed');
|
||||
});
|
||||
|
||||
test('state 5: clean-probe ERROR with readable HEAD → changed (fail toward work)', () => {
|
||||
_setGitHeadProbeForTests(() => 'abc123');
|
||||
_setGitCleanProbeForTests(() => null);
|
||||
expect(probeSourceGitState('/tmp/repo', 'abc123', { requireCleanWorkingTree: true }))
|
||||
.toBe('changed');
|
||||
});
|
||||
|
||||
test('state 6: NULL inputs → changed, head probe never called (case-4 contract)', () => {
|
||||
let probeCalls = 0;
|
||||
_setGitHeadProbeForTests(() => { probeCalls++; return 'abc'; });
|
||||
expect(probeSourceGitState(null, 'abc')).toBe('changed');
|
||||
expect(probeSourceGitState('/tmp/repo', null)).toBe('changed');
|
||||
expect(probeSourceGitState('', '')).toBe('changed');
|
||||
expect(probeCalls).toBe(0);
|
||||
});
|
||||
|
||||
test('state 7: boolean façade parity — isSourceUnchangedSinceSync === (state is unchanged)', () => {
|
||||
_setGitHeadProbeForTests(() => 'abc123');
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
for (const [path, commit] of [
|
||||
['/tmp/repo', 'abc123'], // unchanged → true
|
||||
['/tmp/repo', 'other'], // changed → false
|
||||
] as const) {
|
||||
expect(isSourceUnchangedSinceSync(path, commit))
|
||||
.toBe(probeSourceGitState(path, commit) === 'unchanged');
|
||||
}
|
||||
_setGitHeadProbeForTests(() => null); // unavailable → false
|
||||
expect(isSourceUnchangedSinceSync('/tmp/gone', 'abc123'))
|
||||
.toBe(probeSourceGitState('/tmp/gone', 'abc123') === 'unchanged');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1580,3 +1580,174 @@ describe('BUG 4 — in-progress sync via live lock, not stale freshness', () =>
|
||||
expect(result.status).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// sync_freshness — clone-unavailable content-lag fallback (stateless deploys)
|
||||
// ============================================================================
|
||||
// A container restart (Docker on EB / K8s / Fly) wipes federated clones;
|
||||
// each one is only re-materialized when that source's next sync job runs.
|
||||
// Until then the LOCAL git short-circuit cannot probe HEAD at all. That is
|
||||
// not evidence of pending work, so instead of falling through to raw
|
||||
// wall-clock age (which no-op syncs never advance → false stale/FAIL for
|
||||
// every quiet source after a restart), the check borrows the REMOTE path's
|
||||
// newest_content_at lag (v0.41.32.0). Contracts:
|
||||
// F1: clone unavailable + content at/before last sync → healthy (lag 0).
|
||||
// F2: clone unavailable + content NEWER than last sync → still stale
|
||||
// (wall-clock) — real missed work is never masked.
|
||||
// F3: clone unavailable + NULL newest_content_at → wall-clock fallback
|
||||
// (pre-migration parity with git short-circuit case 5).
|
||||
// F4: chunker mismatch disables the fallback (D7 — a pending re-chunk is
|
||||
// never masked).
|
||||
// F5: a READABLE clone that failed the short-circuit (HEAD moved) keeps
|
||||
// wall-clock even when newest_content_at is old — the fallback is
|
||||
// scoped to 'unavailable' only.
|
||||
// ============================================================================
|
||||
describe('sync_freshness — clone-unavailable content-lag fallback', () => {
|
||||
function makeStubEngine(rows: any[]): any {
|
||||
return { executeRaw: async () => rows };
|
||||
}
|
||||
function agoMs(ms: number): Date { return new Date(Date.now() - ms); }
|
||||
const HOURS = 60 * 60 * 1000;
|
||||
let currentChunkerVersion: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
const { CHUNKER_VERSION } = await import('../src/core/chunkers/code.ts');
|
||||
currentChunkerVersion = String(CHUNKER_VERSION);
|
||||
_setGitHeadProbeForTests(null);
|
||||
_setGitCleanProbeForTests(null);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(null);
|
||||
_setGitCleanProbeForTests(null);
|
||||
});
|
||||
|
||||
test('F1: quiet source, clone gone, content predates last sync → ok', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(() => null); // clone not re-materialized yet
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'quiet-docs', name: '', local_path: '/tmp/quiet-docs',
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'abc', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(72 * HOURS) }, // content older than last sync
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('ok');
|
||||
expect(result.details).toEqual({
|
||||
unchanged_count: 0, synced_recently_count: 1, stale_count: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test('F2: clone gone but content NEWER than last sync → warn (real work not masked)', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(() => null);
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'missed-work', name: '', local_path: '/tmp/missed-work',
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'abc', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(1 * HOURS) }, // content NEWER than last sync
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toMatch(/40h ago/);
|
||||
expect(result.details?.stale_count).toBe(1);
|
||||
});
|
||||
|
||||
test('F3: clone gone + NULL newest_content_at → wall-clock fallback (warn)', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(() => null);
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'pre-migration', name: '', local_path: '/tmp/pre-migration',
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'abc', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: null },
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.details?.stale_count).toBe(1);
|
||||
});
|
||||
|
||||
test('F4: clone gone + chunker MISMATCH → fallback disabled, wall-clock warn', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(() => null);
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'needs-rechunk', name: '', local_path: '/tmp/needs-rechunk',
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'abc',
|
||||
chunker_version: '0', // STALE — re-chunk pending
|
||||
newest_content_at: agoMs(72 * HOURS) },
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.details?.stale_count).toBe(1);
|
||||
});
|
||||
|
||||
test('F5: readable clone, HEAD moved → wall-clock even with old content', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests(() => 'NEW-HEAD'); // clone readable, real work
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'has-commits', name: '', local_path: '/tmp/has-commits',
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'OLD-HEAD', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(72 * HOURS) },
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('warn');
|
||||
expect(result.message).toMatch(/40h ago/);
|
||||
expect(result.details?.stale_count).toBe(1);
|
||||
});
|
||||
|
||||
test('F6: three-bucket invariant holds across rescued + unchanged + stale', async () => {
|
||||
const { checkSyncFreshness } = await import('../src/commands/doctor.ts');
|
||||
const { _setGitHeadProbeForTests, _setGitCleanProbeForTests } =
|
||||
await import('../src/core/git-head.ts');
|
||||
_setGitHeadProbeForTests((path) => path === '/tmp/frozen' ? 'frozen-sha' : null);
|
||||
_setGitCleanProbeForTests(() => true);
|
||||
|
||||
const result = await checkSyncFreshness(makeStubEngine([
|
||||
{ id: 'frozen', name: '', local_path: '/tmp/frozen', // unchanged bucket
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'frozen-sha', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(80 * HOURS) },
|
||||
{ id: 'rescued', name: '', local_path: '/tmp/rescued', // clone gone, quiet → healthy
|
||||
last_sync_at: agoMs(40 * HOURS),
|
||||
last_commit: 'abc', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(80 * HOURS) },
|
||||
{ id: 'stale', name: '', local_path: '/tmp/stale', // clone gone, content newer → stale
|
||||
last_sync_at: agoMs(5 * 24 * HOURS),
|
||||
last_commit: 'def', chunker_version: currentChunkerVersion,
|
||||
newest_content_at: agoMs(1 * HOURS) },
|
||||
]), { localOnly: true });
|
||||
|
||||
expect(result.status).toBe('fail');
|
||||
expect(result.message).toContain(`'stale'`);
|
||||
expect(result.message).not.toContain(`'rescued'`);
|
||||
expect(result.details).toEqual({
|
||||
unchanged_count: 1, synced_recently_count: 1, stale_count: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160) — LIVE Postgres parity.
|
||||
*
|
||||
* The PGLite coverage lives in test/extraction-review.test.ts; this file
|
||||
* re-runs the SQL-touching pieces on a real Postgres so the shared
|
||||
* `unverifiedExtractionFragment` predicate, the engine method
|
||||
* `getUnverifiedExtractionPageIds`, the source-boost guard inside
|
||||
* `buildSourceFactorCase`, and the review-op raw SQL are proven on both
|
||||
* engines (PGLite can hide postgres.js-specific behavior).
|
||||
*
|
||||
* Gated by DATABASE_URL via hasDatabase(); skips cleanly when unset.
|
||||
*
|
||||
* Run: DATABASE_URL=... bun test test/e2e/extraction-review-postgres.test.ts
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import type { PostgresEngine } from '../../src/core/postgres-engine.ts';
|
||||
import { hasDatabase, setupDB, teardownDB } from './helpers.ts';
|
||||
import { enrichEntity } from '../../src/core/enrichment-service.ts';
|
||||
import { isUnverifiedExtraction, STATUS_VERIFIED, EXTRACTION_STATUS_KEY } from '../../src/core/extraction-review.ts';
|
||||
import { operationsByName, type OperationContext } from '../../src/core/operations.ts';
|
||||
|
||||
const RUN = hasDatabase();
|
||||
const d = RUN ? describe : describe.skip;
|
||||
|
||||
let engine: PostgresEngine;
|
||||
|
||||
function ctx(over: Partial<OperationContext> = {}): OperationContext {
|
||||
return {
|
||||
engine,
|
||||
config: {} as OperationContext['config'],
|
||||
logger: { info() {}, warn() {}, error() {}, debug() {} } as unknown as OperationContext['logger'],
|
||||
dryRun: false,
|
||||
remote: true,
|
||||
sourceId: 'default',
|
||||
...over,
|
||||
} as OperationContext;
|
||||
}
|
||||
|
||||
d('extraction quarantine lane (live Postgres)', () => {
|
||||
beforeAll(async () => {
|
||||
engine = await setupDB();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await teardownDB();
|
||||
}, 60_000);
|
||||
|
||||
test('untrusted enrichEntity → markers; getUnverifiedExtractionPageIds sees them', async () => {
|
||||
await enrichEntity(engine, { entityName: 'Pg Fake', entityType: 'person', context: 'c', sourceSlug: 's' });
|
||||
await enrichEntity(engine, { entityName: 'Pg Real', entityType: 'person', context: 'c', sourceSlug: 's' }, { trusted: true });
|
||||
const fake = await engine.getPage('people/pg-fake');
|
||||
const real = await engine.getPage('people/pg-real');
|
||||
expect(isUnverifiedExtraction(fake!.frontmatter)).toBe(true);
|
||||
expect(isUnverifiedExtraction(real!.frontmatter)).toBe(false);
|
||||
const set = await engine.getUnverifiedExtractionPageIds([fake!.id, real!.id]);
|
||||
expect(set.has(fake!.id)).toBe(true);
|
||||
expect(set.has(real!.id)).toBe(false);
|
||||
});
|
||||
|
||||
test('SQL source-boost guard: unverified stub loses the people/ 1.2x in searchKeyword', async () => {
|
||||
await engine.upsertChunks('people/pg-fake', [{ chunk_index: 0, chunk_text: 'flurbo synergy report alpha', chunk_source: 'compiled_truth', token_count: 4 }]);
|
||||
await engine.upsertChunks('people/pg-real', [{ chunk_index: 0, chunk_text: 'flurbo synergy report bravo', chunk_source: 'compiled_truth', token_count: 4 }]);
|
||||
const rows = await engine.searchKeyword('flurbo', { limit: 10 });
|
||||
const fake = rows.find((r) => r.slug === 'people/pg-fake')!;
|
||||
const real = rows.find((r) => r.slug === 'people/pg-real')!;
|
||||
expect(fake).toBeDefined();
|
||||
expect(real).toBeDefined();
|
||||
// Same base ts_rank; only the verified page carries the 1.2 factor.
|
||||
expect(real.score / fake.score).toBeCloseTo(1.2, 5);
|
||||
});
|
||||
|
||||
test('vector arm: unverified stub gets source factor 1.0 in searchVector re-rank', async () => {
|
||||
// The 1.2x people/ factor multiplies raw_score inside the scored CTE,
|
||||
// pre-LIMIT — the guard column projected in hnsw_candidates must zero it
|
||||
// out for unverified stubs. Identical basis embeddings → identical
|
||||
// cosine → the score ratio IS the factor. (1536-dim basis vectors match
|
||||
// the shared e2e schema, same as test/e2e/engine-parity.test.ts.)
|
||||
const basis = new Float32Array(1536);
|
||||
basis[7] = 1.0;
|
||||
await engine.upsertChunks('people/pg-fake', [{ chunk_index: 1, chunk_text: 'vec alpha', chunk_source: 'compiled_truth', embedding: basis, token_count: 2 }]);
|
||||
await engine.upsertChunks('people/pg-real', [{ chunk_index: 1, chunk_text: 'vec bravo', chunk_source: 'compiled_truth', embedding: basis, token_count: 2 }]);
|
||||
const rows = await engine.searchVector(basis, { limit: 10 });
|
||||
const fake = rows.find((r) => r.slug === 'people/pg-fake')!;
|
||||
const real = rows.find((r) => r.slug === 'people/pg-real')!;
|
||||
expect(fake).toBeDefined();
|
||||
expect(real).toBeDefined();
|
||||
expect(real.score / fake.score).toBeCloseTo(1.2, 5);
|
||||
});
|
||||
|
||||
test('extraction_pending + extraction_review promote/reject run on Postgres', async () => {
|
||||
const pending = (await operationsByName['extraction_pending']!.handler(ctx(), {})) as {
|
||||
pending: Array<{ slug: string }>;
|
||||
};
|
||||
expect(pending.pending.map((r) => r.slug)).toContain('people/pg-fake');
|
||||
|
||||
const out = (await operationsByName['extraction_review']!.handler(ctx({ remote: false }), {
|
||||
action: 'promote', slugs: ['people/pg-fake'],
|
||||
})) as { results: Array<{ slug: string; status: string }> };
|
||||
expect(out.results[0].status).toBe('promoted');
|
||||
const promoted = await engine.getPage('people/pg-fake');
|
||||
expect(promoted!.frontmatter[EXTRACTION_STATUS_KEY]).toBe(STATUS_VERIFIED);
|
||||
|
||||
await enrichEntity(engine, { entityName: 'Pg Reject', entityType: 'person', context: 'c', sourceSlug: 's' });
|
||||
const rej = (await operationsByName['extraction_review']!.handler(ctx({ remote: false }), {
|
||||
action: 'reject', slugs: ['people/pg-reject'],
|
||||
})) as { results: Array<{ slug: string; status: string }> };
|
||||
expect(rej.results[0].status).toBe('rejected');
|
||||
expect(await engine.getPage('people/pg-reject')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -39,10 +39,53 @@ import {
|
||||
NON_EXTRACTABLE_AUDIT_SOURCE,
|
||||
PER_SEGMENT_SOURCE_PREFIX,
|
||||
ALLOWED_TYPES,
|
||||
pageTypesForAllowed,
|
||||
ALLOWED_TYPE_ALIASES,
|
||||
} from '../src/commands/extract-conversation-facts.ts';
|
||||
import { _resetLlmCacheForTests } from '../src/core/conversation-parser/llm-base.ts';
|
||||
import { BudgetExhausted } from '../src/core/budget/budget-tracker.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// pageTypesForAllowed — logical→concrete page-type expansion.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('pageTypesForAllowed', () => {
|
||||
test('expands slack to canonical + granular collector types', () => {
|
||||
expect(pageTypesForAllowed(['slack'])).toEqual(['slack', 'slack-dm-day', 'slack-thread']);
|
||||
});
|
||||
|
||||
test('expands email to canonical + granular collector types', () => {
|
||||
expect(pageTypesForAllowed(['email'])).toEqual(['email', 'email-digest']);
|
||||
});
|
||||
|
||||
test('canonical-only types pass through unchanged', () => {
|
||||
expect(pageTypesForAllowed(['meeting'])).toEqual(['meeting']);
|
||||
expect(pageTypesForAllowed(['conversation'])).toEqual(['conversation']);
|
||||
});
|
||||
|
||||
test('canonical name is always first so consolidated brains keep working', () => {
|
||||
expect(pageTypesForAllowed(['slack'])[0]).toBe('slack');
|
||||
expect(pageTypesForAllowed(['email'])[0]).toBe('email');
|
||||
});
|
||||
|
||||
test('multiple logical types flatten and de-duplicate', () => {
|
||||
const got = pageTypesForAllowed(['slack', 'email', 'meeting']);
|
||||
expect(got).toEqual(['slack', 'slack-dm-day', 'slack-thread', 'email', 'email-digest', 'meeting']);
|
||||
// no duplicates
|
||||
expect(new Set(got).size).toBe(got.length);
|
||||
});
|
||||
|
||||
test('every ALLOWED_TYPE_ALIASES entry lists its canonical name first', () => {
|
||||
for (const [canonical, concretes] of Object.entries(ALLOWED_TYPE_ALIASES)) {
|
||||
expect(concretes[0]).toBe(canonical);
|
||||
}
|
||||
});
|
||||
|
||||
test('empty input yields empty output', () => {
|
||||
expect(pageTypesForAllowed([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture helpers.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
/**
|
||||
* Extraction quarantine lane (issue #160).
|
||||
*
|
||||
* `extractAndEnrich` regex-extracts entity names from arbitrary ingested text
|
||||
* and creates people/ + companies/ stub pages. These tests pin the lane
|
||||
* end-to-end:
|
||||
* - fail-closed trust: only an explicit `trusted: true` (which the op layer
|
||||
* only grants for ctx.remote === false AND --trusted-extraction) writes
|
||||
* authoritative pages; undefined/false/remote → quarantine markers.
|
||||
* - unverified stubs are excluded from authoritative retrieval boosts
|
||||
* (compiled-truth fusion boost + the SQL namespace source-boost) and
|
||||
* carry `unverified: true` in search-result metadata.
|
||||
* - review queue: extraction_pending lists; extraction_review promotes
|
||||
* (status → verified) / rejects (soft-delete) in batch, owner-only.
|
||||
* - doctor nudge: unverified_extractions counts stale stubs.
|
||||
*
|
||||
* Hermetic via PGLite (both engines share the SQL through
|
||||
* unverifiedExtractionFragment + the same literal method SQL; postgres runs
|
||||
* via the DATABASE_URL-gated e2e lane).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
|
||||
import {
|
||||
quarantineMarkers,
|
||||
isUnverifiedExtraction,
|
||||
unverifiedExtractionFragment,
|
||||
EXTRACTION_STATUS_KEY,
|
||||
STATUS_UNVERIFIED,
|
||||
STATUS_VERIFIED,
|
||||
PROVENANCE_AUTO_EXTRACTED,
|
||||
} from '../src/core/extraction-review.ts';
|
||||
import { enrichEntity, extractAndEnrich } from '../src/core/enrichment-service.ts';
|
||||
import { rrfFusion, hybridSearch } from '../src/core/search/hybrid.ts';
|
||||
import { buildSourceFactorCase } from '../src/core/search/sql-ranking.ts';
|
||||
import { operationsByName, OperationError, type OperationContext } from '../src/core/operations.ts';
|
||||
import { checkUnverifiedExtractions } from '../src/commands/doctor.ts';
|
||||
import { categorizeCheck } from '../src/core/doctor-categories.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
function basisEmbedding(idx: number, dim = 1536): Float32Array {
|
||||
const emb = new Float32Array(dim);
|
||||
emb[idx % dim] = 1.0;
|
||||
return emb;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// Deterministic no-embedding-provider path: configure the gateway with NO
|
||||
// auth env so hybridSearch never attempts a real embedding call, even on a
|
||||
// dev machine with provider keys in process.env. Pins the vector dim too
|
||||
// (shard-order defense, same class as doctor-hidden-by-search-policy).
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: {},
|
||||
});
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw('DELETE FROM content_chunks');
|
||||
await engine.executeRaw('DELETE FROM links');
|
||||
await engine.executeRaw('DELETE FROM timeline_entries');
|
||||
await engine.executeRaw('DELETE FROM pages');
|
||||
});
|
||||
|
||||
function ctx(over: Partial<OperationContext> = {}): OperationContext {
|
||||
return {
|
||||
engine,
|
||||
config: {} as OperationContext['config'],
|
||||
logger: { info() {}, warn() {}, error() {}, debug() {} } as unknown as OperationContext['logger'],
|
||||
dryRun: false,
|
||||
remote: true,
|
||||
sourceId: 'default',
|
||||
...over,
|
||||
} as OperationContext;
|
||||
}
|
||||
|
||||
/** ctx with `remote` deleted entirely — the type-bypass case the fail-closed
|
||||
* invariant exists for ("anything not strictly false is untrusted"). */
|
||||
function ctxNoRemote(): OperationContext {
|
||||
const c = ctx() as unknown as Record<string, unknown>;
|
||||
delete c.remote;
|
||||
return c as unknown as OperationContext;
|
||||
}
|
||||
|
||||
const extract_entities = operationsByName['extract_entities']!;
|
||||
const extraction_pending = operationsByName['extraction_pending']!;
|
||||
const extraction_review = operationsByName['extraction_review']!;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Marker module (pure)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('extraction-review markers', () => {
|
||||
test('quarantineMarkers → provenance + status pair', () => {
|
||||
expect(quarantineMarkers()).toEqual({ provenance: PROVENANCE_AUTO_EXTRACTED, status: STATUS_UNVERIFIED });
|
||||
});
|
||||
|
||||
test('isUnverifiedExtraction requires BOTH markers', () => {
|
||||
expect(isUnverifiedExtraction(quarantineMarkers())).toBe(true);
|
||||
expect(isUnverifiedExtraction({ status: 'unverified' })).toBe(false);
|
||||
expect(isUnverifiedExtraction({ provenance: 'auto-extracted' })).toBe(false);
|
||||
expect(isUnverifiedExtraction({ provenance: 'auto-extracted', status: 'verified' })).toBe(false);
|
||||
expect(isUnverifiedExtraction({ status: 'unverified', provenance: 'user' })).toBe(false);
|
||||
expect(isUnverifiedExtraction(null)).toBe(false);
|
||||
expect(isUnverifiedExtraction(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
test('SQL fragment references both keys on the given alias', () => {
|
||||
const frag = unverifiedExtractionFragment('p');
|
||||
expect(frag).toContain("p.frontmatter");
|
||||
expect(frag).toContain(PROVENANCE_AUTO_EXTRACTED);
|
||||
expect(frag).toContain(STATUS_UNVERIFIED);
|
||||
});
|
||||
|
||||
test('buildSourceFactorCase guards unverified stubs in both forms', () => {
|
||||
const qualified = buildSourceFactorCase('p.slug', { 'people/': 1.2 }, 'low');
|
||||
expect(qualified).toContain(unverifiedExtractionFragment('p'));
|
||||
expect(qualified.indexOf(unverifiedExtractionFragment('p'))).toBeLessThan(qualified.indexOf('people/'));
|
||||
// Vector re-rank form: bare slug column + pre-computed guard column
|
||||
// (projected in hnsw_candidates) — the guard WHEN must come first.
|
||||
const guarded = buildSourceFactorCase('slug', { 'people/': 1.2 }, 'low', 'unverified_stub');
|
||||
expect(guarded).toContain('CASE WHEN unverified_stub THEN 1.0');
|
||||
expect(guarded.indexOf('unverified_stub')).toBeLessThan(guarded.indexOf('people/'));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fusion boost skip (pure)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('rrfFusion compiled-truth boost skip', () => {
|
||||
function result(slug: string, over: Partial<SearchResult> = {}): SearchResult {
|
||||
return {
|
||||
slug,
|
||||
page_id: over.page_id ?? 1,
|
||||
title: slug,
|
||||
type: 'person',
|
||||
chunk_text: 'x',
|
||||
chunk_source: 'compiled_truth',
|
||||
chunk_id: over.chunk_id ?? 1,
|
||||
chunk_index: 0,
|
||||
score: 1,
|
||||
stale: false,
|
||||
...over,
|
||||
} as SearchResult;
|
||||
}
|
||||
|
||||
test('unverified compiled_truth chunk does NOT get the 2x boost', () => {
|
||||
const verified = result('people/real', { page_id: 1, chunk_id: 1 });
|
||||
const unverified = result('people/fake', { page_id: 2, chunk_id: 2, unverified: true });
|
||||
// Two single-result lists at the same rank → identical raw RRF scores.
|
||||
const fused = rrfFusion([[verified], [unverified]], 60, true);
|
||||
const v = fused.find((r) => r.slug === 'people/real')!;
|
||||
const u = fused.find((r) => r.slug === 'people/fake')!;
|
||||
expect(u.unverified).toBe(true);
|
||||
// Same normalized base; verified gets 2.0x, unverified stays 1.0x.
|
||||
expect(v.score).toBeCloseTo(u.score * 2.0, 10);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Enrichment write path (PGLite)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('enrichEntity trust lane', () => {
|
||||
test('default (opts omitted) → fail-closed quarantine markers', async () => {
|
||||
const r = await enrichEntity(engine, {
|
||||
entityName: 'Mallory Fake',
|
||||
entityType: 'person',
|
||||
context: 'injected sentence',
|
||||
sourceSlug: 'inbox/hostile-email',
|
||||
});
|
||||
expect(r.action).toBe('created');
|
||||
expect(r.quarantined).toBe(true);
|
||||
const page = await engine.getPage('people/mallory-fake');
|
||||
expect(isUnverifiedExtraction(page!.frontmatter)).toBe(true);
|
||||
expect(page!.frontmatter[EXTRACTION_STATUS_KEY]).toBe(STATUS_UNVERIFIED);
|
||||
});
|
||||
|
||||
test('trusted: true → direct authoritative write, no markers', async () => {
|
||||
const r = await enrichEntity(engine, {
|
||||
entityName: 'Alice Example',
|
||||
entityType: 'person',
|
||||
context: 'my own notes',
|
||||
sourceSlug: 'notes/daily',
|
||||
}, { trusted: true });
|
||||
expect(r.action).toBe('created');
|
||||
expect(r.quarantined).toBeUndefined();
|
||||
const page = await engine.getPage('people/alice-example');
|
||||
expect(isUnverifiedExtraction(page!.frontmatter)).toBe(false);
|
||||
expect(page!.frontmatter[EXTRACTION_STATUS_KEY]).toBeUndefined();
|
||||
});
|
||||
|
||||
test('trusted: false explicitly → quarantine markers', async () => {
|
||||
await enrichEntity(engine, {
|
||||
entityName: 'Widget Co Corp',
|
||||
entityType: 'company',
|
||||
context: 'ctx',
|
||||
sourceSlug: 'inbox/x',
|
||||
}, { trusted: false });
|
||||
const page = await engine.getPage('companies/widget-co-corp');
|
||||
expect(isUnverifiedExtraction(page!.frontmatter)).toBe(true);
|
||||
});
|
||||
|
||||
test('vector arm: unverified stub gets source factor 1.0, not the people/ 1.2x', async () => {
|
||||
// The 1.2x namespace factor is applied INSIDE searchVector's re-rank SQL,
|
||||
// pre-LIMIT — an unguarded stub would outrank AND could evict legitimate
|
||||
// pages from the candidate pool before fusion ever sees them. Identical
|
||||
// basis embeddings → identical cosine → the score ratio IS the factor.
|
||||
await enrichEntity(engine, { entityName: 'Vec Fake', entityType: 'person', context: 'c', sourceSlug: 's' });
|
||||
await enrichEntity(engine, { entityName: 'Vec Real', entityType: 'person', context: 'c', sourceSlug: 's' }, { trusted: true });
|
||||
const e = basisEmbedding(7);
|
||||
await engine.upsertChunks('people/vec-fake', [{ chunk_index: 0, chunk_text: 'vector text alpha', chunk_source: 'compiled_truth', embedding: e, token_count: 3 }]);
|
||||
await engine.upsertChunks('people/vec-real', [{ chunk_index: 0, chunk_text: 'vector text bravo', chunk_source: 'compiled_truth', embedding: e, token_count: 3 }]);
|
||||
const rows = await engine.searchVector(e, { limit: 10 });
|
||||
const fake = rows.find((r) => r.slug === 'people/vec-fake')!;
|
||||
const real = rows.find((r) => r.slug === 'people/vec-real')!;
|
||||
expect(fake).toBeDefined();
|
||||
expect(real).toBeDefined();
|
||||
expect(real.score / fake.score).toBeCloseTo(1.2, 5);
|
||||
});
|
||||
|
||||
test('getUnverifiedExtractionPageIds returns only marked pages', async () => {
|
||||
await enrichEntity(engine, { entityName: 'Fake Guy', entityType: 'person', context: 'c', sourceSlug: 's' });
|
||||
await enrichEntity(engine, { entityName: 'Real Guy', entityType: 'person', context: 'c', sourceSlug: 's' }, { trusted: true });
|
||||
const fake = await engine.getPage('people/fake-guy');
|
||||
const real = await engine.getPage('people/real-guy');
|
||||
const set = await engine.getUnverifiedExtractionPageIds([fake!.id, real!.id]);
|
||||
expect(set.has(fake!.id)).toBe(true);
|
||||
expect(set.has(real!.id)).toBe(false);
|
||||
expect((await engine.getUnverifiedExtractionPageIds([])).size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ops: trust-boundary matrix + review queue
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('extract_entities op trust boundary', () => {
|
||||
const TEXT = 'I had lunch with Bobby Injected today. He said Evil Widgets Inc is pivoting.';
|
||||
|
||||
test('remote: true → quarantined even WITH trusted_extraction flag', async () => {
|
||||
const out = (await extract_entities.handler(ctx({ remote: true }), {
|
||||
text: TEXT, source_slug: 'inbox/mail', trusted_extraction: true,
|
||||
})) as { trusted: boolean; quarantined: number; count: number };
|
||||
expect(out.trusted).toBe(false);
|
||||
expect(out.count).toBeGreaterThan(0);
|
||||
expect(out.quarantined).toBe(out.count);
|
||||
const page = await engine.getPage('people/bobby-injected');
|
||||
expect(isUnverifiedExtraction(page!.frontmatter)).toBe(true);
|
||||
});
|
||||
|
||||
test('remote UNSET (type bypass) → fail-closed quarantine', async () => {
|
||||
const out = (await extract_entities.handler(ctxNoRemote(), {
|
||||
text: TEXT, source_slug: 'inbox/mail', trusted_extraction: true,
|
||||
})) as { trusted: boolean; quarantined: number };
|
||||
expect(out.trusted).toBe(false);
|
||||
expect(out.quarantined).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('remote: false WITHOUT flag → still quarantined (explicit opt-in required)', async () => {
|
||||
const out = (await extract_entities.handler(ctx({ remote: false }), {
|
||||
text: TEXT, source_slug: 'inbox/mail',
|
||||
})) as { trusted: boolean; quarantined: number };
|
||||
expect(out.trusted).toBe(false);
|
||||
expect(out.quarantined).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('resource guards: oversize text rejected; entity flood capped + surfaced', async () => {
|
||||
// Oversize input → loud invalid_params, nothing written.
|
||||
await expect(extract_entities.handler(ctx(), {
|
||||
text: 'A'.repeat(200_001), source_slug: 'inbox/big',
|
||||
})).rejects.toBeInstanceOf(OperationError);
|
||||
// 300 distinct name-shaped tokens → capped at 200, truncated surfaced.
|
||||
// 300 distinct two-word names (letters only — the extractor regex is
|
||||
// [A-Z][a-z]+ per word, digits would break the match).
|
||||
const flood = Array.from({ length: 300 }, (_, i) =>
|
||||
`Flood Name${String.fromCharCode(97 + (i % 26))}${String.fromCharCode(97 + Math.floor(i / 26))}`,
|
||||
).join('. ');
|
||||
const out = (await extract_entities.handler(ctx(), { text: flood, source_slug: 'inbox/flood' })) as {
|
||||
count: number; entities_found: number; truncated: boolean;
|
||||
};
|
||||
expect(out.entities_found).toBeGreaterThan(200);
|
||||
expect(out.count).toBe(200);
|
||||
expect(out.truncated).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test('remote: false WITH --trusted-extraction → direct authoritative write', async () => {
|
||||
const out = (await extract_entities.handler(ctx({ remote: false }), {
|
||||
text: TEXT, source_slug: 'notes/mine', trusted_extraction: true,
|
||||
})) as { trusted: boolean; quarantined: number; count: number };
|
||||
expect(out.trusted).toBe(true);
|
||||
expect(out.quarantined).toBe(0);
|
||||
const page = await engine.getPage('people/bobby-injected');
|
||||
expect(isUnverifiedExtraction(page!.frontmatter)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extraction_pending + extraction_review', () => {
|
||||
async function seedStub(name: string): Promise<string> {
|
||||
const r = await enrichEntity(engine, { entityName: name, entityType: 'person', context: 'c', sourceSlug: 'inbox/x' });
|
||||
return r.slug;
|
||||
}
|
||||
|
||||
test('pending lists unverified stubs; promoted/rejected drop out', async () => {
|
||||
const a = await seedStub('Fake Aa');
|
||||
const b = await seedStub('Fake Bb');
|
||||
await enrichEntity(engine, { entityName: 'Real Cc', entityType: 'person', context: 'c', sourceSlug: 's' }, { trusted: true });
|
||||
|
||||
const before = (await extraction_pending.handler(ctx(), {})) as { count: number; pending: Array<{ slug: string }> };
|
||||
expect(before.pending.map((r) => r.slug).sort()).toEqual([a, b].sort());
|
||||
|
||||
const out = (await extraction_review.handler(ctx({ remote: false }), {
|
||||
action: 'promote', slugs: [a],
|
||||
})) as { results: Array<{ slug: string; status: string }> };
|
||||
expect(out.results).toEqual([{ slug: a, status: 'promoted' }]);
|
||||
|
||||
const promoted = await engine.getPage(a);
|
||||
expect(promoted!.frontmatter[EXTRACTION_STATUS_KEY]).toBe(STATUS_VERIFIED);
|
||||
// provenance survives as the audit trail.
|
||||
expect(promoted!.frontmatter.provenance).toBe(PROVENANCE_AUTO_EXTRACTED);
|
||||
expect(isUnverifiedExtraction(promoted!.frontmatter)).toBe(false);
|
||||
|
||||
const rej = (await extraction_review.handler(ctx({ remote: false }), {
|
||||
action: 'reject', slugs: b, // CLI string form
|
||||
})) as { results: Array<{ slug: string; status: string }> };
|
||||
expect(rej.results).toEqual([{ slug: b, status: 'rejected' }]);
|
||||
expect(await engine.getPage(b)).toBeNull(); // soft-deleted → hidden
|
||||
|
||||
const after = (await extraction_pending.handler(ctx(), {})) as { count: number };
|
||||
expect(after.count).toBe(0);
|
||||
});
|
||||
|
||||
test('batch promote is batch-friendly and reports per-slug statuses', async () => {
|
||||
const a = await seedStub('Fake Dd');
|
||||
const b = await seedStub('Fake Ee');
|
||||
await enrichEntity(engine, { entityName: 'Real Ff', entityType: 'person', context: 'c', sourceSlug: 's' }, { trusted: true });
|
||||
const out = (await extraction_review.handler(ctx({ remote: false }), {
|
||||
action: 'promote', slugs: [a, b, 'people/real-ff', 'people/missing'],
|
||||
})) as { results: Array<{ slug: string; status: string }> };
|
||||
expect(out.results.map((r) => r.status)).toEqual(['promoted', 'promoted', 'not_unverified', 'not_found']);
|
||||
});
|
||||
|
||||
test('extraction_review is owner-only: remote and unset-trust callers are refused', async () => {
|
||||
const a = await seedStub('Fake Gg');
|
||||
await expect(extraction_review.handler(ctx({ remote: true }), { action: 'promote', slugs: [a] }))
|
||||
.rejects.toBeInstanceOf(OperationError);
|
||||
await expect(extraction_review.handler(ctxNoRemote(), { action: 'promote', slugs: [a] }))
|
||||
.rejects.toBeInstanceOf(OperationError);
|
||||
// and it is not exposed over HTTP MCP at all
|
||||
expect(extraction_review.localOnly).toBe(true);
|
||||
// stub untouched
|
||||
expect(isUnverifiedExtraction((await engine.getPage(a))!.frontmatter)).toBe(true);
|
||||
});
|
||||
|
||||
test('invalid action / empty slugs → invalid_params', async () => {
|
||||
await expect(extraction_review.handler(ctx({ remote: false }), { action: 'bless', slugs: ['x'] }))
|
||||
.rejects.toBeInstanceOf(OperationError);
|
||||
await expect(extraction_review.handler(ctx({ remote: false }), { action: 'promote', slugs: [] }))
|
||||
.rejects.toBeInstanceOf(OperationError);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Doctor nudge
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('unverified_extractions doctor check', () => {
|
||||
test('fresh stubs → ok; stale stubs → warn with review commands', async () => {
|
||||
await enrichEntity(engine, { entityName: 'Fake Hh', entityType: 'person', context: 'c', sourceSlug: 's' });
|
||||
const fresh = await checkUnverifiedExtractions(engine);
|
||||
expect(fresh.status).toBe('ok');
|
||||
|
||||
await engine.executeRaw(`UPDATE pages SET created_at = now() - interval '30 days' WHERE slug = 'people/fake-hh'`);
|
||||
const stale = await checkUnverifiedExtractions(engine, { days: 7 });
|
||||
expect(stale.status).toBe('warn');
|
||||
expect(stale.message).toContain('extraction-pending');
|
||||
expect(stale.message).toContain('extraction-review');
|
||||
expect((stale.details as { count: number }).count).toBe(1);
|
||||
});
|
||||
|
||||
test('categorized as a brain check', () => {
|
||||
expect(categorizeCheck('unverified_extractions')).toBe('brain');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// End-to-end: hostile transcript → quarantined stubs, NOT boosted in search
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('e2e: hostile transcript', () => {
|
||||
test('fake entities land quarantined and rank without entity authority', async () => {
|
||||
// 1. Hostile transcript arrives through an agent-facing (remote) caller.
|
||||
const transcript =
|
||||
'Meeting notes. I had lunch with Zorbulon Fakeperson today. ' +
|
||||
'He mentioned the zorbulon pivot is confirmed.';
|
||||
const out = (await extract_entities.handler(ctx({ remote: true }), {
|
||||
text: transcript, source_slug: 'meetings/2026-04-03', trusted_extraction: true,
|
||||
})) as { trusted: boolean; quarantined: number };
|
||||
expect(out.trusted).toBe(false);
|
||||
expect(out.quarantined).toBeGreaterThan(0);
|
||||
|
||||
const stub = await engine.getPage('people/zorbulon-fakeperson');
|
||||
expect(stub).not.toBeNull();
|
||||
expect(isUnverifiedExtraction(stub!.frontmatter)).toBe(true);
|
||||
|
||||
// 2. Owner-authored control page with the same lexical relevance.
|
||||
await engine.putPage('people/zorbulon-realperson', {
|
||||
type: 'person', title: 'Zorbulon Realperson', compiled_truth: 'zorbulon notes', timeline: '', frontmatter: {},
|
||||
});
|
||||
const real = await engine.getPage('people/zorbulon-realperson');
|
||||
|
||||
// 3. Chunk both with equal lexical relevance (distinct texts — identical
|
||||
// ones would be Jaccard-deduped). Enrichment stubs are chunked by the
|
||||
// normal reindex/import pipeline later; seed what it would write.
|
||||
await engine.upsertChunks(stub!.slug, [{ chunk_index: 0, chunk_text: 'zorbulon pivot details from the injected meeting', chunk_source: 'compiled_truth', token_count: 7 }]);
|
||||
await engine.upsertChunks(real!.slug, [{ chunk_index: 0, chunk_text: 'zorbulon launch update in my own written notes', chunk_source: 'compiled_truth', token_count: 7 }]);
|
||||
|
||||
// 4. Search. No embedding provider configured → keyword(+title) fusion path.
|
||||
const results = await hybridSearch(engine, 'zorbulon', { limit: 10 });
|
||||
const fake = results.find((r) => r.slug === stub!.slug);
|
||||
const legit = results.find((r) => r.slug === real!.slug);
|
||||
expect(fake).toBeDefined();
|
||||
expect(legit).toBeDefined();
|
||||
|
||||
// Clearly marked in search-result metadata…
|
||||
expect(fake!.unverified).toBe(true);
|
||||
expect(legit!.unverified).toBeUndefined();
|
||||
// …and stripped of entity authority: the verified page outranks the
|
||||
// injected stub despite identical chunk text (2x compiled-truth boost +
|
||||
// people/ source-boost apply only to the verified page).
|
||||
expect(legit!.score).toBeGreaterThan(fake!.score);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* #3026: thin-client `jobs list`/`get` receive MinionJob rows as parsed JSON
|
||||
* off the MCP wire — every timestamp an ISO string — while formatJob /
|
||||
* formatJobDetail and the stalled-detection comparison hold a Date contract
|
||||
* (locally hydrated by MinionQueue.rowToJob). Before the fix, `jobs get <id>`
|
||||
* on a thin client crashed with "job.started_at.toISOString is not a
|
||||
* function" the moment the remote routing actually worked (unmasked by
|
||||
* #2951's scratch-engine fix).
|
||||
*
|
||||
* Pins rehydrateJobDates (the unpack-boundary coercion) plus, audit-style,
|
||||
* that both thin-client unpack sites route through it.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { rehydrateJobDates } from '../src/commands/jobs.ts';
|
||||
|
||||
describe('rehydrateJobDates', () => {
|
||||
test('coerces wire-format ISO strings to Dates on all timestamp fields', () => {
|
||||
const wire = {
|
||||
id: 1192,
|
||||
name: 'autopilot-cycle',
|
||||
status: 'completed',
|
||||
created_at: '2026-07-21T04:02:11.512Z',
|
||||
updated_at: '2026-07-21T04:02:14.930Z',
|
||||
started_at: '2026-07-21T04:02:12.001Z',
|
||||
finished_at: '2026-07-21T04:02:14.900Z',
|
||||
lock_until: '2026-07-21T04:03:12.001Z',
|
||||
delay_until: null,
|
||||
};
|
||||
const job = rehydrateJobDates(wire);
|
||||
expect(job.created_at).toBeInstanceOf(Date);
|
||||
expect(job.updated_at).toBeInstanceOf(Date);
|
||||
expect(job.started_at).toBeInstanceOf(Date);
|
||||
expect(job.finished_at).toBeInstanceOf(Date);
|
||||
expect(job.lock_until).toBeInstanceOf(Date);
|
||||
expect((job.started_at as unknown as Date).toISOString()).toBe('2026-07-21T04:02:12.001Z');
|
||||
// Date math used by formatJob's duration column works post-rehydration.
|
||||
expect((job.finished_at as unknown as Date).getTime() - (job.started_at as unknown as Date).getTime())
|
||||
.toBeCloseTo(2899, 0);
|
||||
});
|
||||
|
||||
test('leaves Dates, nulls, and non-timestamp fields untouched', () => {
|
||||
const started = new Date('2026-07-21T04:02:12.001Z');
|
||||
const job = rehydrateJobDates({
|
||||
id: 7,
|
||||
name: 'sync',
|
||||
status: 'active',
|
||||
created_at: started,
|
||||
started_at: started,
|
||||
finished_at: null,
|
||||
delay_until: undefined,
|
||||
});
|
||||
expect(job.created_at).toBe(started);
|
||||
expect(job.finished_at).toBeNull();
|
||||
expect(job.delay_until).toBeUndefined();
|
||||
expect(job.name).toBe('sync');
|
||||
});
|
||||
|
||||
test('does not fabricate Dates from malformed strings; passes null through', () => {
|
||||
const job = rehydrateJobDates({ id: 8, started_at: 'not-a-date' });
|
||||
expect(job.started_at).toBe('not-a-date');
|
||||
expect(rehydrateJobDates(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('thin-client unpack sites route through rehydrateJobDates (source audit)', () => {
|
||||
const src = readFileSync(join(import.meta.dir, '..', 'src', 'commands', 'jobs.ts'), 'utf8');
|
||||
|
||||
test('list branch rehydrates', () => {
|
||||
expect(src).toContain('unpackToolResult<MinionJob[]>(raw).map((j) => rehydrateJobDates(j))');
|
||||
});
|
||||
|
||||
test('get branch rehydrates', () => {
|
||||
expect(src).toContain('rehydrateJobDates(unpackToolResult<MinionJob | null>(raw))');
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
inferLinkType,
|
||||
makeResolver,
|
||||
parseTimelineEntries,
|
||||
deriveTimelineAnchor,
|
||||
isAutoLinkEnabled,
|
||||
FRONTMATTER_LINK_MAP,
|
||||
unwrapWikilink,
|
||||
@@ -852,6 +853,55 @@ More prose here.
|
||||
});
|
||||
});
|
||||
|
||||
// ─── deriveTimelineAnchor ──────────────────────────────────────
|
||||
|
||||
describe('deriveTimelineAnchor', () => {
|
||||
test('anchors at a frontmatter effective_date with the page title as summary', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'meetings/2026-04-24-handover',
|
||||
title: 'Ops handover',
|
||||
effectiveDate: new Date('2026-04-24T09:00:00Z'),
|
||||
effectiveDateSource: 'event_date',
|
||||
});
|
||||
expect(a).toEqual({ date: '2026-04-24', summary: 'Ops handover', detail: '' });
|
||||
});
|
||||
|
||||
test('accepts a filename-sourced date and an ISO-string effectiveDate', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'daily/2022-04-20-standup',
|
||||
title: '',
|
||||
effectiveDate: '2022-04-20',
|
||||
effectiveDateSource: 'filename',
|
||||
});
|
||||
expect(a).toEqual({ date: '2022-04-20', summary: '2022-04-20-standup', detail: '' });
|
||||
});
|
||||
|
||||
test('returns null for the fallback (updated_at) source — not a real content date', () => {
|
||||
expect(deriveTimelineAnchor({
|
||||
slug: 'notes/x', title: 'X',
|
||||
effectiveDate: new Date('2026-01-01T00:00:00Z'),
|
||||
effectiveDateSource: 'fallback',
|
||||
})).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null when no date or no source', () => {
|
||||
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: null, effectiveDateSource: 'date' })).toBeNull();
|
||||
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: new Date('2026-01-01Z'), effectiveDateSource: null })).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null on an unparseable date string', () => {
|
||||
expect(deriveTimelineAnchor({ slug: 'a', title: 'A', effectiveDate: 'not-a-date', effectiveDateSource: 'date' })).toBeNull();
|
||||
});
|
||||
|
||||
test('falls back to the slug basename when title is empty', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'people/jane-example-com', title: ' ',
|
||||
effectiveDate: '2025-12-31', effectiveDateSource: 'published',
|
||||
});
|
||||
expect(a?.summary).toBe('jane-example-com');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── isAutoLinkEnabled ─────────────────────────────────────────
|
||||
|
||||
function makeFakeEngine(configMap: Map<string, string | null>): BrainEngine {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* Regression guard for scripts/check-no-tracked-symlinks.sh.
|
||||
*
|
||||
* Commit faf5cdba tracked `node_modules -> /tmp/fleet/repo/node_modules`.
|
||||
* That path exists on one build sandbox and nowhere else, so every other
|
||||
* clone got a dangling symlink and `bun install` aborted with
|
||||
* `ENOENT: could not open the "node_modules" directory` — which also took
|
||||
* out `gbrain upgrade`'s bun-link path, since it shells out to bun install.
|
||||
*
|
||||
* `.gitignore` did not stop it: a `node_modules/` pattern with a trailing
|
||||
* slash matches directories only, so the symlink was never ignored. The
|
||||
* pattern is fixed, but `git add -f` still bypasses .gitignore entirely,
|
||||
* so the shell guard is the real backstop. These tests pin (1) the guard
|
||||
* detects a tracked symlink, (2) it stays green on this repo, and (3) it
|
||||
* is actually wired into `bun run verify`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'bun:test';
|
||||
import { existsSync, statSync, mkdtempSync, rmSync, writeFileSync, symlinkSync } from 'fs';
|
||||
import { resolve, join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..');
|
||||
const GUARD = resolve(REPO_ROOT, 'scripts/check-no-tracked-symlinks.sh');
|
||||
const VERIFY_DISPATCHER = resolve(REPO_ROOT, 'scripts/run-verify-parallel.sh');
|
||||
|
||||
describe('check-no-tracked-symlinks.sh', () => {
|
||||
it('exists and is executable', () => {
|
||||
expect(existsSync(GUARD)).toBe(true);
|
||||
expect((statSync(GUARD).mode & 0o100) !== 0).toBe(true);
|
||||
});
|
||||
|
||||
it('passes on this repo (no tracked symlinks)', () => {
|
||||
const r = spawnSync('bash', [GUARD], { cwd: REPO_ROOT, encoding: 'utf-8' });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('OK');
|
||||
});
|
||||
|
||||
it('fails and names the offender when a symlink is tracked', () => {
|
||||
// Build a throwaway repo rather than poisoning this one's index.
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gbrain-symlink-guard-'));
|
||||
try {
|
||||
const git = (...args: string[]) =>
|
||||
spawnSync('git', args, { cwd: dir, encoding: 'utf-8' });
|
||||
|
||||
git('init', '-q');
|
||||
git('config', 'user.email', 'test@example.com');
|
||||
git('config', 'user.name', 'test');
|
||||
|
||||
writeFileSync(join(dir, 'README.md'), '# fixture\n');
|
||||
// Absolute target that does not exist — the exact shape of the bug.
|
||||
symlinkSync('/tmp/does-not-exist/node_modules', join(dir, 'node_modules'));
|
||||
git('add', '-A');
|
||||
|
||||
const r = spawnSync('bash', [GUARD], { cwd: dir, encoding: 'utf-8' });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stdout).toContain('node_modules -> /tmp/does-not-exist/node_modules');
|
||||
expect(r.stdout).toContain('git rm --cached');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('is wired into the verify dispatcher', () => {
|
||||
const r = spawnSync('bash', [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
expect(new Set(r.stdout.trim().split('\n'))).toContain('check:no-tracked-symlinks');
|
||||
});
|
||||
});
|
||||
|
||||
describe('.gitignore node_modules patterns', () => {
|
||||
it('match symlinks too (no trailing slash)', () => {
|
||||
const lines = require('fs')
|
||||
.readFileSync(resolve(REPO_ROOT, '.gitignore'), 'utf-8')
|
||||
.split('\n')
|
||||
.map((l: string) => l.trim())
|
||||
.filter((l: string) => l && !l.startsWith('#'));
|
||||
|
||||
// A trailing slash restricts the pattern to directories, which is how
|
||||
// the symlink slipped through. Every node_modules rule must be bare.
|
||||
const offenders = lines.filter((l: string) => /node_modules\/$/.test(l));
|
||||
expect(offenders).toEqual([]);
|
||||
expect(lines).toContain('node_modules');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
// Regression: no-embedding-provider early-return must be multimodal-aware.
|
||||
//
|
||||
// On a multimodal-only install (text embedding provider ABSENT, a multimodal
|
||||
// provider such as Voyage multimodal-3 PRESENT), hybridSearch's
|
||||
// no-embedding-provider short-circuit used to probe ONLY the text column's
|
||||
// provider. Since that provider is unreachable, search returned to the
|
||||
// keyword-only path (vector_enabled:false) BEFORE the image/unified vector
|
||||
// routing below ever ran — so image and unified queries silently degraded to
|
||||
// keyword search even though a usable multimodal vector path existed.
|
||||
//
|
||||
// The fix adds a `willTryMultimodal` guard that also probes the multimodal
|
||||
// provider so the early-return does not fire when multimodal vectoring is
|
||||
// still possible. These tests assert that the multimodal (Voyage) embedding
|
||||
// endpoint is actually reached on a text-provider-absent install.
|
||||
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
|
||||
import { hybridSearch } from '../src/core/search/hybrid.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let fetchHandler: ((url: string, init: RequestInit) => Promise<Response>) | null = null;
|
||||
const origFetch = globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => {
|
||||
if (!fetchHandler) throw new Error('no fetch handler');
|
||||
return fetchHandler(typeof url === 'string' ? url : url.toString(), init ?? {});
|
||||
}) as typeof fetch;
|
||||
|
||||
// Multimodal-only install: a text embedding model is *configured* but its
|
||||
// required auth env (OPENAI_API_KEY) is ABSENT, so the text provider is
|
||||
// unreachable. The multimodal provider (Voyage) IS reachable (VOYAGE_API_KEY
|
||||
// present). This is exactly the install shape the fix targets.
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
embedding_multimodal_model: 'voyage:voyage-multimodal-3',
|
||||
env: { VOYAGE_API_KEY: 'test' },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = origFetch;
|
||||
resetGateway();
|
||||
fetchHandler = null;
|
||||
});
|
||||
|
||||
describe('multimodal-only install: no-embedding early-return is multimodal-aware', () => {
|
||||
test('image query still reaches the multimodal vector path (does not short-circuit to keyword)', async () => {
|
||||
let voyageCalled = 0;
|
||||
let openaiCalled = 0;
|
||||
fetchHandler = async (url) => {
|
||||
if (url.includes('multimodalembeddings')) {
|
||||
voyageCalled++;
|
||||
return new Response(JSON.stringify({
|
||||
data: [{ embedding: Array.from({ length: 1024 }, () => 0.1), index: 0 }],
|
||||
}), { status: 200 });
|
||||
}
|
||||
if (url.includes('api.openai.com') && url.includes('embeddings')) {
|
||||
openaiCalled++;
|
||||
}
|
||||
return new Response(JSON.stringify({
|
||||
data: [{ embedding: Array.from({ length: 1536 }, () => 0.1), index: 0 }],
|
||||
}), { status: 200 });
|
||||
};
|
||||
|
||||
const results = await hybridSearch(engine, 'a photo of a red bicycle', {
|
||||
limit: 5,
|
||||
crossModal: 'image',
|
||||
});
|
||||
|
||||
// Pre-fix: the text-provider probe failed → early-return → Voyage never
|
||||
// called. Post-fix: the image branch runs and embeds via the multimodal
|
||||
// (Voyage) provider.
|
||||
expect(voyageCalled).toBeGreaterThanOrEqual(1);
|
||||
// The unreachable text provider must never have been dialed.
|
||||
expect(openaiCalled).toBe(0);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
test('unified_multimodal routing reaches the multimodal vector path on a text-provider-absent install', async () => {
|
||||
await engine.setConfig('search.unified_multimodal', 'true');
|
||||
let voyageCalled = 0;
|
||||
let openaiCalled = 0;
|
||||
fetchHandler = async (url) => {
|
||||
if (url.includes('multimodalembeddings')) {
|
||||
voyageCalled++;
|
||||
return new Response(JSON.stringify({
|
||||
data: [{ embedding: Array.from({ length: 1024 }, () => 0.1), index: 0 }],
|
||||
}), { status: 200 });
|
||||
}
|
||||
if (url.includes('api.openai.com') && url.includes('embeddings')) {
|
||||
openaiCalled++;
|
||||
}
|
||||
return new Response(JSON.stringify({
|
||||
data: [{ embedding: Array.from({ length: 1536 }, () => 0.1), index: 0 }],
|
||||
}), { status: 200 });
|
||||
};
|
||||
|
||||
await hybridSearch(engine, 'totally text query', { limit: 5 });
|
||||
|
||||
// Unified routing forces the multimodal endpoint even for a text-shaped
|
||||
// query; pre-fix the early-return fired first and Voyage was never called.
|
||||
expect(voyageCalled).toBeGreaterThanOrEqual(1);
|
||||
expect(openaiCalled).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
escapeLikePattern as topLevelEscapeLikePattern,
|
||||
__test__,
|
||||
} from '../src/core/search/sql-ranking.ts';
|
||||
import { unverifiedExtractionFragment } from '../src/core/extraction-review.ts';
|
||||
import {
|
||||
DEFAULT_SOURCE_BOOSTS,
|
||||
DEFAULT_HARD_EXCLUDES,
|
||||
@@ -87,9 +88,11 @@ describe('buildSourceFactorCase', () => {
|
||||
expect(buildSourceFactorCase('p.slug', {}, 'medium')).toBe('1.0');
|
||||
});
|
||||
|
||||
test('emits a CASE expression for non-high detail', () => {
|
||||
test('emits a CASE expression for non-high detail (unverified guard first — issue #160)', () => {
|
||||
const result = buildSourceFactorCase('p.slug', { 'originals/': 1.5 }, 'medium');
|
||||
expect(result).toBe("(CASE WHEN p.slug LIKE 'originals/%' THEN 1.5 ELSE 1.0 END)");
|
||||
expect(result).toBe(
|
||||
`(CASE WHEN ${unverifiedExtractionFragment('p')} THEN 1.0 WHEN p.slug LIKE 'originals/%' THEN 1.5 ELSE 1.0 END)`,
|
||||
);
|
||||
});
|
||||
|
||||
test('sorts prefixes by length descending so longest-match wins', () => {
|
||||
@@ -119,7 +122,9 @@ describe('buildSourceFactorCase', () => {
|
||||
{ 'good/': 1.5, 'nan/': NaN, 'neg/': -1, 'inf/': Infinity },
|
||||
'medium',
|
||||
);
|
||||
expect(result).toBe("(CASE WHEN p.slug LIKE 'good/%' THEN 1.5 ELSE 1.0 END)");
|
||||
expect(result).toBe(
|
||||
`(CASE WHEN ${unverifiedExtractionFragment('p')} THEN 1.0 WHEN p.slug LIKE 'good/%' THEN 1.5 ELSE 1.0 END)`,
|
||||
);
|
||||
});
|
||||
|
||||
test('uses the supplied slug column reference', () => {
|
||||
|
||||
Reference in New Issue
Block a user