fix(sync): guard putPage 0-row RETURNING + reclassify sync failure copy (#2189)

Root cause of #2189's opaque per-file crash: putPage's INSERT … ON CONFLICT
DO UPDATE … RETURNING can yield 0 rows when brain-local DB state (e.g. a
BEFORE trigger) suppresses the write, and rowToPage(rows[0]) then died with
"undefined is not an object (evaluating 'row.deleted_at')". Both engines now
throw a descriptive error naming the slug + source_id so the failure is
diagnosable per-file instead of an anonymous TypeError.

Also lands the salvageable parts of community PR #2586 (takeover): the
"failed to parse / fix the frontmatter" copy misclassified runtime import
errors as YAML problems — reworded to "failed to import" — plus its
first-code-sync PGLite smoke test.

New regression test reproduces the exact 0-row RETURNING state via a
suppressing trigger: fails pre-fix with the reported crash signature,
passes with the guard.

Co-authored-by: javieraldape <javieraldape@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/commands/sync.ts

@@ -197,7 +197,7 @@ export interface SyncResult {
   /** Pages re-embedded during this sync's auto-embed step. 0 if --no-embed or skipped. */
   embedded: number;
   pagesAffected: string[];
-  failedFiles?: number; // count of parse failures (Bug 9)
+  failedFiles?: number; // count of per-file import/sync failures (Bug 9)
   /**
    * v0.41.13.0 partial-sync fields (only set when status === 'partial').
    *
@@ -3183,7 +3183,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
     await clearOpCheckpoint(engine, ckpt.target);
   };
 
-  // issue #1939 adversarial finding #1: a file that failed to parse (open ledger
+  // issue #1939 adversarial finding #1: a file that failed to import (open ledger
   // row) and is then deleted/renamed-away never re-enters failedFiles and never
   // imports, so its row would never clear and would age doctor to a permanent
   // FAIL. Treat removed paths as resolved so the ledger self-heals.
@@ -3215,9 +3215,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
     } else {
       const fileFailCount = failedFiles.filter(f => isSkippablePath(f.path)).length;
       serr(
-        `\nSync blocked: ${fileFailCount} file(s) failed to parse:\n` +
+        `\nSync blocked: ${fileFailCount} file(s) failed to import:\n` +
         `${codeBreakdown}\n\n` +
-        `Fix the frontmatter and re-run, or use 'gbrain sync --skip-failed' to ` +
+        `Fix the listed file errors and re-run, or use 'gbrain sync --skip-failed' to ` +
         `acknowledge and move on. A file that keeps failing auto-skips after ` +
         `${resolveAutoSkipThreshold()} consecutive syncs.`,
       );
@@ -5355,7 +5355,7 @@ function printSyncResult(result: SyncResult, sink: NodeJS.WriteStream = process.
     case 'dry_run':
       break; // already printed in performSync
     case 'blocked_by_failures':
-      write(`Sync BLOCKED at ${result.toCommit.slice(0, 8)}: ${result.failedFiles ?? 0} file(s) failed to parse.`);
+      write(`Sync BLOCKED at ${result.toCommit.slice(0, 8)}: ${result.failedFiles ?? 0} file(s) failed to import.`);
       write(`  See ~/.gbrain/sync-failures.jsonl for details, or run 'gbrain doctor'.`);
       write(`  Fix the files then re-run 'gbrain sync', or 'gbrain sync --skip-failed' to move on.`);
       break;

src/core/chunkers/code.ts

@@ -18,9 +18,8 @@
  * at runtime.
  */
 
-import { chunkText as recursiveChunk, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from './recursive.ts';
+import { chunkText as recursiveChunk } from './recursive.ts';
 import { buildQualifiedName } from './qualified-names.ts';
-import { estimateEmbeddingTokens } from '../cjk.ts';
 
 // Embed the tree-sitter runtime + per-language grammars as files.
 // `with { type: 'file' }` returns a path (string) at runtime. Bun bundles
@@ -112,15 +111,7 @@ import G_ZIG from '../../assets/wasm/grammars/tree-sitter-zig.wasm' with { type:
 // chunks get the new columns populated. Without this, the v28 backfill
 // gives every existing chunk a search_vector but subsequent Layer 5 AST
 // work would silently no-op.
-//
-// v5: estimated-token hard cap on AST-path chunks (capCodeChunks). A node
-// splitLargeNode can't subdivide (giant single-statement function, huge
-// literal) previously shipped WHOLE regardless of size and could overflow
-// strict per-request embedding-token limits (local llama-server crashes
-// past ~2,050 tokens, measured). Mirrors the markdown
-// chunker's v4 cap; fallback-path chunks are already capped inside
-// recursiveChunk.
-export const CHUNKER_VERSION = 5;
+export const CHUNKER_VERSION = 4;
 
 // Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
 let Parser: typeof import('web-tree-sitter') | null = null;
@@ -717,7 +708,7 @@ export async function chunkCodeTextFull(
     if (chunks.length === 0) {
       return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
     }
-    return { chunks: capCodeChunks(mergeSmallSiblings(chunks, chunkTarget)), edges: rawEdges };
+    return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
   } catch {
     return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
   } finally {
@@ -800,33 +791,6 @@ function mergeSmallSiblings(chunks: CodeChunk[], chunkTarget: number): CodeChunk
   return merged;
 }
 
-/**
- * v5 final safety pass for AST-path chunks: split any chunk whose
- * ESTIMATED embedding tokens (conservative per-char-class heuristic,
- * cjk.ts) exceed DEFAULT_MAX_EST_TOKENS. Reaches chunks the AST logic
- * can't subdivide — splitLargeNode returns [] for nodes with < 2 body
- * children (giant single-statement functions, huge literals), which
- * previously shipped whole at any size.
- *
- * Split pieces inherit the source chunk's metadata verbatim; start/end
- * lines become approximate for pieces after the first. Acceptable —
- * these chunks exist for embedding + retrieval, and the alternative was
- * an embedding request the server rejects (or worse, crashes on).
- */
-function capCodeChunks(chunks: CodeChunk[]): CodeChunk[] {
-  if (chunks.every((c) => estimateEmbeddingTokens(c.text) <= DEFAULT_MAX_EST_TOKENS)) {
-    return chunks;
-  }
-  const out: CodeChunk[] = [];
-  for (const c of chunks) {
-    const pieces = capByEstimatedTokens(c.text, DEFAULT_MAX_EST_TOKENS);
-    for (const piece of pieces) {
-      out.push({ ...c, text: piece, index: out.length, metadata: { ...c.metadata } });
-    }
-  }
-  return out;
-}
-
 function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
   const first = group[0]!;
   const last = group[group.length - 1]!;

src/core/chunkers/recursive.ts

@@ -17,13 +17,7 @@
  * Lossless invariant: non-overlapping portions reassemble to original.
  */
 
-import {
-  countCJKAwareWords,
-  CJK_SENTENCE_DELIMITERS,
-  CJK_CLAUSE_DELIMITERS,
-  charEmbedTokenWeight,
-  estimateEmbeddingTokens,
-} from '../cjk.ts';
+import { countCJKAwareWords, CJK_SENTENCE_DELIMITERS, CJK_CLAUSE_DELIMITERS } from '../cjk.ts';
 
 /**
  * Markdown chunker version. Folded into the per-page chunker_version column
@@ -39,20 +33,8 @@ import {
  * re-embed (not re-chunk) so existing pages pick up the wrapper on the
  * post-upgrade reembed sweep. See
  * `src/core/contextual-retrieval-service.ts`.
- *
- * v4: estimated-token hard cap + whitespace-word undercount fix. The word
- * pipeline counted a 150-char URL as ONE whitespace word, so URL/phone/
- * email-dense docs (CJK density < 0.30 → whitespace fallback) produced
- * 3-4K-char chunks that overflow strict per-request embedding-token
- * limits (measured: local llama-server crashes past ~2,050 tokens; URL
- * soup tokenizes at ~1.6 chars/token). Two changes:
- *   1. countWords() floors the count at ceil(nonWhitespaceChars/6) so a
- *      URL counts roughly per-character, not as one word.
- *   2. capByEstimatedTokens() final pass guarantees every chunk fits
- *      `maxTokens` (default 1500) under a conservative per-char-class
- *      token estimate, regardless of how word counting misjudged it.
  */
-export const MARKDOWN_CHUNKER_VERSION = 4;
+export const MARKDOWN_CHUNKER_VERSION = 3;
 
 const DELIMITERS: string[][] = [
   ['\n\n'],                          // L0: paragraphs
@@ -66,20 +48,8 @@ export interface ChunkOptions {
   chunkSize?: number;    // target words per chunk (default 300)
   chunkOverlap?: number; // overlap words (default 50)
   maxChars?: number;     // hard cap on any chunk's char length (default 6000)
-  /**
-   * v4: hard cap on any chunk's ESTIMATED embedding tokens (default 1500).
-   * Estimate = conservative per-char-class weights (see cjk.ts
-   * estimateEmbeddingTokens) — deliberately high, so the real tokenizer
-   * count stays below this value. Default leaves headroom for the
-   * contextual-retrieval wrapper (≤ ~630 chars) under a ~2,050-token
-   * per-request embedding server limit.
-   */
-  maxTokens?: number;
 }
 
-/** v4 default for ChunkOptions.maxTokens — see the field doc above. */
-export const DEFAULT_MAX_EST_TOKENS = 1500;
-
 export interface TextChunk {
   text: string;
   index: number;
@@ -103,7 +73,6 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   const chunkSize = opts?.chunkSize || 300;
   const chunkOverlap = opts?.chunkOverlap || 50;
   const maxChars = opts?.maxChars || 6000;
-  const maxTokens = opts?.maxTokens || DEFAULT_MAX_EST_TOKENS;
 
   if (!text || text.trim().length === 0) return [];
 
@@ -120,9 +89,8 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
 
   const wordCount = countWords(stripped);
   if (wordCount <= chunkSize) {
-    // Single-chunk path: still apply the maxChars + maxTokens caps.
-    const capped = capByChars(stripped.trim(), maxChars)
-      .flatMap((t) => capByEstimatedTokens(t, maxTokens));
+    // Single-chunk path: still apply the maxChars cap.
+    const capped = capByChars(stripped.trim(), maxChars);
     return capped.map((t, i) => ({ text: t, index: i }));
   }
 
@@ -133,14 +101,9 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   // v0.32.7: hard char cap. Catches pathological CJK + whitespace-less text
   // that the word-level pipeline can't bound (a single Chinese paragraph can
   // exceed 8192 OpenAI embedding tokens at any word count).
-  // v4: estimated-token cap on top — the char cap alone passes token-dense
-  // content (URL soup at ~1.6 chars/token) that overflows strict embedding
-  // server limits.
   const capped: string[] = [];
   for (const chunk of withOverlap) {
-    for (const piece of capByChars(chunk.trim(), maxChars)) {
-      capped.push(...capByEstimatedTokens(piece, maxTokens));
-    }
+    capped.push(...capByChars(chunk.trim(), maxChars));
   }
   return capped.map((t, i) => ({ text: t, index: i }));
 }
@@ -169,68 +132,6 @@ function capByChars(text: string, maxChars: number): string[] {
   return out;
 }
 
-/**
- * How far back (in chars) the token cap looks for a friendly cut point
- * before falling back to a hard cut. 300 covers typical rollup/list line
- * lengths so forced splits land at line starts, not mid-URL.
- */
-const TOKEN_CAP_CUT_LOOKBACK = 300;
-
-/**
- * v4: hard-cap a chunk's ESTIMATED embedding tokens. Final safety pass —
- * runs after capByChars on every chunk, so no upstream miscounting
- * (whitespace-word fallback, overlap inflation, char-cap survivors) can
- * emit a chunk past `maxTokens`.
- *
- * Cut placement prefers, within the last TOKEN_CAP_CUT_LOOKBACK chars of
- * the window: a newline, then any whitespace, then a hard cut. This keeps
- * forced splits off mid-line/mid-URL positions for list-shaped content
- * and inside code fences. No overlap is added (pieces stay lossless
- * modulo the trims the char cap already applies).
- *
- * @internal exported for the code chunker (code.ts) and tests.
- */
-export function capByEstimatedTokens(text: string, maxTokens: number): string[] {
-  if (text.length === 0) return [];
-  if (estimateEmbeddingTokens(text) <= maxTokens) return [text];
-
-  const out: string[] = [];
-  let start = 0;
-  while (start < text.length) {
-    // Greedily extend the window until the next char would break the cap.
-    // Always take at least one char so the loop makes forward progress.
-    let est = 0;
-    let end = start;
-    while (end < text.length) {
-      const w = charEmbedTokenWeight(text.charCodeAt(end));
-      if (est + w > maxTokens && end > start) break;
-      est += w;
-      end++;
-    }
-
-    if (end < text.length) {
-      const windowStart = Math.max(start + 1, end - TOKEN_CAP_CUT_LOOKBACK);
-      let cut = text.lastIndexOf('\n', end - 1);
-      if (cut < windowStart) {
-        cut = -1;
-        for (let i = end - 1; i >= windowStart; i--) {
-          const code = text.charCodeAt(i);
-          if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) {
-            cut = i;
-            break;
-          }
-        }
-      }
-      if (cut >= windowStart) end = cut + 1;
-    }
-
-    const slice = text.slice(start, end).trim();
-    if (slice.length > 0) out.push(slice);
-    start = end;
-  }
-  return out;
-}
-
 function recursiveSplit(text: string, level: number, target: number): string[] {
   if (level >= DELIMITERS.length) {
     // Level 4: split on whitespace
@@ -416,19 +317,7 @@ function extractTrailingContext(text: string, targetWords: number): string {
  * Delegated to src/core/cjk.ts so the slugify whitelist, expansion
  * detection, and PGLite keyword fallback all agree on what "CJK enough"
  * means.
- *
- * v4: floored at ceil(nonWhitespaceChars/6). The whitespace fallback
- * counts a 150-char URL as ONE word, so URL/phone/email-dense docs
- * (whose ASCII mass pushes CJK density below the 0.30 threshold) were
- * sized at a fraction of their real bulk and merged into 3-4K-char
- * chunks. The floor makes long whitespace-less runs count roughly
- * per-character while leaving normal Latin prose untouched (average
- * English word ≈ 5 chars < 6, so the whitespace count still wins).
- * Kept local to the chunker — search/expansion.ts keeps the original
- * countCJKAwareWords semantics for its query-length check.
  */
 function countWords(text: string): number {
-  const cjkAware = countCJKAwareWords(text);
-  const nonWhitespace = text.replace(/\s/g, '').length;
-  return Math.max(cjkAware, Math.ceil(nonWhitespace / 6));
+  return countCJKAwareWords(text);
 }

src/core/cjk.ts

@@ -65,64 +65,3 @@ export function countCJKAwareWords(s: string): number {
 export function escapeLikePattern(s: string): string {
   return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
 }
-
-/**
- * Conservative per-char-class embedding-token weights (markdown chunker v4).
- *
- * Why this exists: the chunker's "word" counting drastically UNDER-counts
- * whitespace-less ASCII runs (a 150-char URL = 1 whitespace word), so
- * word-based size targets can emit chunks that overflow an embedding
- * server's per-request token limit. Measured on a local Qwen3-embedding
- * llama-server stack:
- *   - URL/phone/email-dense text tokenizes at ~1.6 chars/token
- *   - base64-ish / minified blobs approach ~1.3 chars/token (worst case)
- *   - Korean prose tokenizes NO WORSE than 1 char/token in practice
- *
- * Weights are deliberately HIGH (tokens are overestimated) so any cap
- * based on this estimate is safe against real tokenizers:
- *   - CJK char        → 1.0 token  (real CJK prose is cheaper)
- *   - other non-space → 0.75 token (≈1.33 chars/token, covers base64)
- *   - whitespace      → 0.1 token  (mostly folds into neighbor tokens)
- */
-export const EMBED_TOKEN_WEIGHT_CJK = 1.0;
-export const EMBED_TOKEN_WEIGHT_OTHER = 0.75;
-export const EMBED_TOKEN_WEIGHT_WS = 0.1;
-
-/** BMP CJK check by UTF-16 code unit — same ranges as CJK_SLUG_CHARS. */
-export function isCJKCodeUnit(code: number): boolean {
-  return (
-    (code >= 0x4e00 && code <= 0x9fff) || // Han
-    (code >= 0x3040 && code <= 0x309f) || // Hiragana
-    (code >= 0x30a0 && code <= 0x30ff) || // Katakana
-    (code >= 0xac00 && code <= 0xd7af)    // Hangul Syllables
-  );
-}
-
-/**
- * Per-code-unit token weight. Unrecognized whitespace (exotic Unicode
- * spaces) intentionally falls into OTHER — that only overestimates.
- */
-export function charEmbedTokenWeight(code: number): number {
-  if (isCJKCodeUnit(code)) return EMBED_TOKEN_WEIGHT_CJK;
-  if (
-    code === 0x20 || (code >= 0x09 && code <= 0x0d) ||
-    code === 0xa0 || code === 0x3000
-  ) {
-    return EMBED_TOKEN_WEIGHT_WS;
-  }
-  return EMBED_TOKEN_WEIGHT_OTHER;
-}
-
-/**
- * Tokenizer-free embedding-token estimate (conservative overestimate).
- * See weight docs above. Astral chars count as 2 OTHER code units —
- * another overestimate, which is the safe direction.
- */
-export function estimateEmbeddingTokens(s: string): number {
-  if (s.length === 0) return 0;
-  let est = 0;
-  for (let i = 0; i < s.length; i++) {
-    est += charEmbedTokenWeight(s.charCodeAt(i));
-  }
-  return Math.ceil(est);
-}

src/core/pglite-engine.ts

@@ -1058,6 +1058,16 @@ export class PGLiteEngine implements BrainEngine {
        RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`,
       [sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
     );
+    // #2189: an INSERT … ON CONFLICT DO UPDATE … RETURNING that yields 0 rows
+    // (e.g. a BEFORE trigger suppressing the write) previously crashed in
+    // rowToPage with an opaque "undefined is not an object (row.deleted_at)".
+    // Throw a diagnosable error naming the row instead. Mirrors postgres-engine.ts.
+    if (!rows[0]) {
+      throw new Error(
+        `putPage: INSERT … RETURNING produced no row for slug='${slug}' source_id='${sourceId}'. ` +
+        `A trigger or rule on the pages table may be suppressing the write.`
+      );
+    }
     return rowToPage(rows[0] as Record<string, unknown>);
   }
 

src/core/postgres-engine.ts

@@ -1119,6 +1119,16 @@ export class PostgresEngine implements BrainEngine {
         ingested_at           = COALESCE(EXCLUDED.ingested_at,           pages.ingested_at)
       RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at
     `;
+    // #2189: an INSERT … ON CONFLICT DO UPDATE … RETURNING that yields 0 rows
+    // (e.g. a BEFORE trigger suppressing the write) previously crashed in
+    // rowToPage with an opaque "undefined is not an object (row.deleted_at)".
+    // Throw a diagnosable error naming the row instead. Mirrors pglite-engine.ts.
+    if (!rows[0]) {
+      throw new Error(
+        `putPage: INSERT … RETURNING produced no row for slug='${slug}' source_id='${sourceId}'. ` +
+        `A trigger or rule on the pages table may be suppressing the write.`
+      );
+    }
     return rowToPage(rows[0]);
   }
 

test/chunker-version-gate.test.ts

@@ -15,22 +15,19 @@ import { describe, test, expect } from 'bun:test';
 import { CHUNKER_VERSION } from '../src/core/chunkers/code.ts';
 
 describe('Layer 12 — CHUNKER_VERSION constant', () => {
-  test('bumped to 5 for the estimated-token hard cap', () => {
+  test('bumped to 4 for Cathedral II', () => {
     // v3: v0.19.0 Chonkie parity (tokenizer + small-sibling merge).
     // v4: v0.20.0 Cathedral II (qualified names + parent scope + doc_comment
     //     + fence extraction + chunk-grain FTS). Folded into content_hash
     //     so any bump forces clean re-chunks on next sync.
-    // v5: estimated-token hard cap on AST-path chunks (capCodeChunks) so
-    //     un-subdividable giant nodes can't overflow strict embedding
-    //     server token limits.
-    expect(CHUNKER_VERSION).toBe(5);
+    expect(CHUNKER_VERSION).toBe(4);
   });
 
   test('is stable across imports (not recomputed at call time)', async () => {
     const a = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     const b = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     expect(a).toBe(b);
-    expect(a).toBe(5);
+    expect(a).toBe(4);
   });
 });
 

test/chunkers/code.test.ts

@@ -10,8 +10,8 @@ import { describe, test, expect } from 'bun:test';
 import { chunkCodeText, detectCodeLanguage, CHUNKER_VERSION } from '../../src/core/chunkers/code.ts';
 
 describe('CHUNKER_VERSION', () => {
-  test('v5: estimated-token hard cap on AST-path chunks', () => {
-    expect(CHUNKER_VERSION).toBe(5);
+  test('v0.20.0 Cathedral II Layer 12 bumped to 4', () => {
+    expect(CHUNKER_VERSION).toBe(4);
   });
 });
 

test/chunkers/recursive.test.ts

@@ -135,14 +135,13 @@ describe('Recursive Text Chunker', () => {
 });
 
 describe('CJK chunking (v0.32.7)', () => {
-  test('MARKDOWN_CHUNKER_VERSION is 4', async () => {
+  test('MARKDOWN_CHUNKER_VERSION is 3', async () => {
     // v0.40.3.0: bumped 2→3 to signal the post-upgrade reembed sweep that
-    // contextual retrieval wrapping is now applied at embed time.
-    // v4: estimated-token hard cap + whitespace-word undercount floor
-    // (URL-dense docs produced chunks past strict embedding server token
-    // limits). Boundary change → forces re-chunk for chunker_version < 4.
+    // contextual retrieval wrapping is now applied at embed time. Chunk
+    // boundaries themselves are unchanged; the bump forces re-embed for
+    // pages where chunker_version < 3.
     const mod = await import('../../src/core/chunkers/recursive.ts');
-    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(4);
+    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(3);
   });
 
   test('long pure-Chinese paragraph splits into multiple chunks', () => {

test/chunkers/token-cap.test.ts

@@ -1,195 +0,0 @@
-/**
- * Markdown chunker v4 / code chunker v5 — estimated-token hard cap
- * regression tests.
- *
- * Reproduces a field failure: a local llama-server embedding backend
- * (`-ub 2048`) crashes deterministically (trace/BPT trap → EOF at the
- * client) when a single chunk exceeds ~2,050 real tokens. Two content
- * shapes triggered it:
- *
- *   1. Korean docs carrying one long source URL per line.
- *      The URLs' ASCII mass pushes CJK density below 0.30, flipping
- *      countCJKAwareWords to whitespace counting, where a 150-char URL
- *      counts as ONE word → chunks ballooned to 3-4K chars ≈ 2,000+
- *      real tokens (URL soup tokenizes at ~1.6 chars/token).
- *
- *   2. Large JSON code blocks (~7K chars) that the word pipeline
- *      undercounts the same way (few whitespace tokens).
- *
- * The fix: every emitted chunk must satisfy
- *   estimateEmbeddingTokens(chunk) <= maxTokens (default 1500)
- * where the estimate deliberately OVERSTATES real tokenizer counts.
- */
-
-import { describe, test, expect } from 'bun:test';
-import { chunkText, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from '../../src/core/chunkers/recursive.ts';
-import { chunkCodeText } from '../../src/core/chunkers/code.ts';
-import { estimateEmbeddingTokens } from '../../src/core/cjk.ts';
-
-/** Synthesize the failing shape: Korean rollup lines each ending in a long Notion URL. */
-function urlDenseKoreanRollup(lines: number): string {
-  const out: string[] = ['# 링크가 줄마다 붙는 한국어 예시 문서', ''];
-  for (let i = 0; i < lines; i++) {
-    const hex32 = (i * 2654435761 >>> 0).toString(16).padStart(8, '0').repeat(4);
-    out.push(
-      `- **항목 ${i}**: 이 줄은 청커 동작 검증을 위한 의미 없는 한국어 예시 문장입니다 · 전화 000-0000-${String(1000 + i)} · ` +
-      `이메일 user${i}@example.com · 링크: https://docs.example.com/pages/${hex32}?v=abcdef0123456789&ref=sample`,
-    );
-  }
-  return out.join('\n');
-}
-
-/** Synthesize a large pretty-printed JSON block with CJK values. */
-function bigJsonBlock(targetChars: number): string {
-  const entries: string[] = [];
-  let i = 0;
-  let len = 0;
-  while (len < targetChars) {
-    const row =
-      `  "item_${i}": { "name": "예시-${i}", "url": "https://example.com/api/v2/items/${i}?token=abc${i}def", "qty": ${i % 100}, "memo": "한국어 값이 섞인 예시 데이터" }`;
-    entries.push(row);
-    len += row.length;
-    i++;
-  }
-  return `{\n${entries.join(',\n')}\n}`;
-}
-
-describe('v4 estimated-token cap — URL-dense Korean doc (field-failure shape)', () => {
-  test('every chunk stays under the estimated-token cap', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('no chunk reaches the measured 3K-char danger zone for URL soup', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    // 1500 est tokens at the OTHER weight (0.75/char) bounds chunks to
-    // ~2,000 chars for pure ASCII — well under the ~3,300 chars where
-    // URL-dense content crosses ~2,050 real tokens (1.6 chars/token).
-    for (const c of chunks) {
-      expect(c.text.length).toBeLessThanOrEqual(2600);
-    }
-  });
-
-  test('content is preserved (no lines dropped by the cap)', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    const joined = chunks.map((c) => c.text).join('\n');
-    // Spot-check first / middle / last rollup lines survive chunking.
-    for (const marker of ['항목 0', '항목 30', '항목 59']) {
-      expect(joined).toContain(marker);
-    }
-  });
-});
-
-describe('v4 estimated-token cap — large JSON blocks', () => {
-  test('7K-char pretty JSON through the prose path stays under the cap', () => {
-    const md = `설정 파일 원문 보존:\n\n\`\`\`\n${bigJsonBlock(7000)}\n\`\`\`\n`;
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('7K-char minified JSON (single whitespace-less token) stays under the cap', () => {
-    const minified = bigJsonBlock(7000).replace(/\n\s*/g, '');
-    const chunks = chunkText(minified);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('json fence via the code chunker stays under the cap (+header slack)', async () => {
-    const chunks = await chunkCodeText(bigJsonBlock(7000), 'fence.json');
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      // buildChunk prepends a short "[JSON] fence.json:…" header AFTER the
-      // body-level cap; allow ~60 est tokens of header slack. Real-token
-      // safety margin (2,050 − overestimated 1,500) absorbs this easily.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS + 60);
-    }
-  });
-});
-
-describe('v4 word-count floor — behavior preserved for normal content', () => {
-  test('Latin prose chunking is unchanged by the floor (avg word < 6 chars)', () => {
-    const prose = Array.from({ length: 120 }, (_, i) =>
-      `This is sentence number ${i} and it talks about ordinary things in plain words.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    // Historical behavior: ~1,560 whitespace words → multiple ~300-word chunks.
-    expect(chunks.length).toBeGreaterThan(3);
-    for (const c of chunks) {
-      const words = c.text.split(/\s+/).length;
-      expect(words).toBeLessThanOrEqual(300 * 1.5 + 50); // merge cap + overlap
-    }
-  });
-
-  test('Korean prose (CJK-dense, no URLs) never triggers the token cap', () => {
-    const prose = Array.from({ length: 80 }, (_, i) =>
-      `이 문장은 순수 한국어 산문의 청킹 동작을 확인하기 위한 ${i}번째 예시 문장입니다.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      // CJK-dense chunks are char-counted (≈450 max) — nowhere near 1500.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(700);
-    }
-  });
-});
-
-describe('capByEstimatedTokens unit behavior', () => {
-  test('returns input unchanged when under the cap', () => {
-    expect(capByEstimatedTokens('short text', 1500)).toEqual(['short text']);
-    expect(capByEstimatedTokens('', 1500)).toEqual([]);
-  });
-
-  test('prefers newline cut points within the lookback window', () => {
-    const line = 'x'.repeat(100);
-    const text = Array.from({ length: 40 }, () => line).join('\n');
-    const pieces = capByEstimatedTokens(text, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    for (const p of pieces) {
-      // Every piece should be whole lines (multiples of the 100-char line).
-      for (const l of p.split('\n')) {
-        expect(l).toBe(line);
-      }
-    }
-  });
-
-  test('makes forward progress on whitespace-less input (hard cut)', () => {
-    const blob = 'a'.repeat(10_000);
-    const pieces = capByEstimatedTokens(blob, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    expect(pieces.join('')).toBe(blob);
-    for (const p of pieces) {
-      expect(estimateEmbeddingTokens(p)).toBeLessThanOrEqual(1000);
-    }
-  });
-});
-
-describe('estimateEmbeddingTokens — weight sanity', () => {
-  test('overestimates URL-dense ASCII (0.75/char ≥ measured ~0.63/char)', () => {
-    const url = 'https://docs.example.com/pages/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4?v=abc&ref=sample';
-    const est = estimateEmbeddingTokens(url);
-    expect(est).toBeGreaterThanOrEqual(Math.floor(url.length * 0.7));
-  });
-
-  test('counts CJK at 1 token/char', () => {
-    expect(estimateEmbeddingTokens('가나다라마')).toBe(5);
-  });
-
-  test('whitespace is nearly free', () => {
-    expect(estimateEmbeddingTokens('   \n\t  ')).toBeLessThanOrEqual(1);
-  });
-
-  test('empty string is 0', () => {
-    expect(estimateEmbeddingTokens('')).toBe(0);
-  });
-});

test/putpage-returning-guard.test.ts

@@ -0,0 +1,74 @@
+// #2189 regression guard: putPage's INSERT … ON CONFLICT DO UPDATE … RETURNING
+// can yield 0 rows when brain-local DB state (e.g. a BEFORE INSERT trigger)
+// suppresses the write. Pre-fix, rowToPage(rows[0]) crashed with the opaque
+// "undefined is not an object (evaluating 'row.deleted_at')" that failed
+// ~all files of a code sync. Post-fix, putPage throws a descriptive error
+// naming the slug + source_id so the failure is diagnosable per-file.
+//
+// Same guard lands in postgres-engine.ts (engine-parity invariant); this test
+// exercises the PGLite side, where the issue was reported.
+
+import { describe, expect, test, beforeAll, afterAll } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+
+let engine: PGLiteEngine;
+
+beforeAll(async () => {
+  engine = new PGLiteEngine();
+  await engine.connect({});
+  await engine.initSchema();
+  // Simulate the reporter's state-dependent failure: a trigger that
+  // suppresses inserts for one slug, making RETURNING produce no row.
+  await engine.executeRaw(`
+    CREATE OR REPLACE FUNCTION suppress_pages_insert() RETURNS trigger AS $$
+    BEGIN
+      IF NEW.slug = 'suppressed-page' THEN RETURN NULL; END IF;
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
+  await engine.executeRaw(`
+    CREATE TRIGGER suppress_pages_insert_trg
+    BEFORE INSERT ON pages
+    FOR EACH ROW EXECUTE FUNCTION suppress_pages_insert();
+  `);
+});
+
+afterAll(async () => {
+  await engine.executeRaw('DROP TRIGGER IF EXISTS suppress_pages_insert_trg ON pages');
+  await engine.executeRaw('DROP FUNCTION IF EXISTS suppress_pages_insert');
+  await engine.disconnect();
+});
+
+describe('putPage RETURNING guard (#2189)', () => {
+  test('0-row RETURNING throws a descriptive error, not row.deleted_at TypeError', async () => {
+    let err: Error | undefined;
+    try {
+      await engine.putPage('suppressed-page', {
+        type: 'code',
+        title: 'Suppressed',
+        compiled_truth: 'x',
+        timeline: '',
+      });
+    } catch (e) {
+      err = e as Error;
+    }
+    expect(err).toBeDefined();
+    expect(err!.message).toContain('putPage');
+    expect(err!.message).toContain("slug='suppressed-page'");
+    expect(err!.message).toContain("source_id='default'");
+    // The pre-fix crash signature must be gone.
+    expect(err!.message).not.toContain('deleted_at');
+  });
+
+  test('unsuppressed slugs still upsert normally with the trigger installed', async () => {
+    const page = await engine.putPage('normal-page', {
+      type: 'concept',
+      title: 'Normal',
+      compiled_truth: 'y',
+      timeline: '',
+    });
+    expect(page.slug).toBe('normal-page');
+    expect(page.source_id).toBe('default');
+  });
+});

test/sync.test.ts

@@ -375,6 +375,31 @@ describe('performSync dry-run never writes', () => {
     expect(messages.some(m => m.includes('git pull failed'))).toBe(false);
   });
 
+  test('first PGLite code sync imports code files without runtime failures', async () => {
+    const { performSync } = await import('../src/commands/sync.ts');
+    mkdirSync(join(repoPath, 'src'), { recursive: true });
+    writeFileSync(
+      join(repoPath, 'src/example.ts'),
+      'export function add(left: number, right: number) { return left + right; }\n',
+    );
+    execSync('git add -A && git commit -m "add code file"', { cwd: repoPath, stdio: 'pipe' });
+
+    const result = await performSync(engine, {
+      repoPath,
+      noPull: true,
+      noEmbed: true,
+      noExtract: true,
+      strategy: 'code',
+    });
+
+    expect(result.status).toBe('first_sync');
+    expect(result.added).toBe(1);
+    expect(result.failedFiles ?? 0).toBe(0);
+    const page = await engine.getPage('src-example-ts');
+    expect(page?.type).toBe('code');
+    expect(page?.frontmatter).toMatchObject({ file: 'src/example.ts', language: 'typescript' });
+  });
+
   test('incremental dry-run does NOT write to DB or advance the bookmark', async () => {
     const { performSync } = await import('../src/commands/sync.ts');
     // First do a real sync to seed the bookmark.