From 84fad4738d3f8c627ab75b48a77624c9ca9ddcd7 Mon Sep 17 00:00:00 2001 From: Hanchen Qiu <131798003+JOJOCrazy123@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:47:09 +0800 Subject: [PATCH] fix(pages): default chunker_version to MARKDOWN_CHUNKER_VERSION on INSERT (#2807) (#2988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit putPage's INSERT used COALESCE(, 1), so callers that don't supply chunker_version (no MCP/subagent caller does — it's internal metadata) landed new pages at version 1. Dream subagents write through putPage directly, so their pages got v1 and doctor's contextual_retrieval_coverage check flagged them as "older chunker_version" forever, even though they were chunked and embedded with the current chunker. Default the INSERT to MARKDOWN_CHUNKER_VERSION on both engines. The ON CONFLICT UPDATE still COALESCE-preserves an explicitly supplied version. Add an engine-level regression test. --- src/core/pglite-engine.ts | 3 +- src/core/postgres-engine.ts | 3 +- test/chunker-version-insert-default.test.ts | 80 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 test/chunker-version-insert-default.test.ts diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index d084079af..3b3a9cf01 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -23,6 +23,7 @@ import { runMigrations } from './migrate.ts'; import { PGLITE_SCHEMA_SQL, getPGLiteSchema } from './pglite-schema.ts'; import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts'; import { DELETE_BATCH_SIZE } from './engine-constants.ts'; +import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts'; import { acquireLock, releaseLock, type LockHandle } from './pglite-lock.ts'; import { getFtsLanguage } from './fts-language.ts'; import type { @@ -1035,7 +1036,7 @@ export class PGLiteEngine implements BrainEngine { const ingestedAt = (sourceKind || sourceUri || ingestedVia) ? new Date().toISOString() : null; const { rows } = await this.db.query( `INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12, COALESCE($13, 1), $14, $15, $16, $17, $18::timestamptz) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12, COALESCE($13, ${MARKDOWN_CHUNKER_VERSION}), $14, $15, $16, $17, $18::timestamptz) ON CONFLICT (source_id, slug) DO UPDATE SET type = EXCLUDED.type, page_kind = EXCLUDED.page_kind, diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 071c69b4e..8e014bf48 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -35,6 +35,7 @@ import { EmbeddingColumnNotRegisteredError, } from './search/embedding-column.ts'; import { getFtsLanguage } from './fts-language.ts'; +import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts'; import type { Page, PageInput, PageFilters, PageType, Chunk, ChunkInput, StaleChunkRow, StalePageRow, @@ -1097,7 +1098,7 @@ export class PostgresEngine implements BrainEngine { const ingestedAt = (sourceKind || sourceUri || ingestedVia) ? new Date() : null; const rows = await sql` INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at) - VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}, COALESCE(${chunkerVersion}::smallint, 1), ${sourcePath}, ${sourceKind}, ${sourceUri}, ${ingestedVia}, ${ingestedAt}) + VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}, COALESCE(${chunkerVersion}::smallint, ${MARKDOWN_CHUNKER_VERSION}), ${sourcePath}, ${sourceKind}, ${sourceUri}, ${ingestedVia}, ${ingestedAt}) ON CONFLICT (source_id, slug) DO UPDATE SET type = EXCLUDED.type, page_kind = EXCLUDED.page_kind, diff --git a/test/chunker-version-insert-default.test.ts b/test/chunker-version-insert-default.test.ts new file mode 100644 index 000000000..86980ee45 --- /dev/null +++ b/test/chunker-version-insert-default.test.ts @@ -0,0 +1,80 @@ +/** + * #2807 — putPage INSERT default for chunker_version. + * + * postgres-engine / pglite-engine used `COALESCE(, 1)` in the + * pages INSERT. Callers that don't supply chunker_version (no MCP/subagent + * caller does — it's internal metadata) landed new pages at version 1, so + * doctor's contextual_retrieval_coverage check flagged dream-written pages as + * "older chunker_version" forever, even though they were chunked/embedded with + * the current chunker. + * + * Fix: default the INSERT to MARKDOWN_CHUNKER_VERSION. The ON CONFLICT UPDATE + * still COALESCE-preserves an explicitly supplied version. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { MARKDOWN_CHUNKER_VERSION } from '../src/core/chunkers/recursive.ts'; + +describe('#2807 — putPage chunker_version INSERT default', () => { + let engine: PGLiteEngine; + + beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + }); + + afterAll(async () => { + await engine.disconnect(); + }, 30_000); + + async function readChunkerVersion(slug: string): Promise { + const { rows } = await (engine as any).db.query( + `SELECT chunker_version FROM pages WHERE slug = $1`, + [slug], + ); + return Number((rows[0] as { chunker_version: number }).chunker_version); + } + + test('page written without chunker_version defaults to MARKDOWN_CHUNKER_VERSION, not 1', async () => { + await engine.putPage('dream/no-chunker-version', { + type: 'concept', + title: 'Dream synthesized page', + compiled_truth: 'Written directly through putPage, like a dream subagent.', + timeline: '', + }); + expect(await readChunkerVersion('dream/no-chunker-version')).toBe(MARKDOWN_CHUNKER_VERSION); + expect(MARKDOWN_CHUNKER_VERSION).toBeGreaterThan(1); + }); + + test('explicit chunker_version is honored on INSERT', async () => { + await engine.putPage('dream/explicit-chunker-version', { + type: 'concept', + title: 'Explicit version', + compiled_truth: 'Caller supplied a version.', + timeline: '', + chunker_version: 2, + }); + expect(await readChunkerVersion('dream/explicit-chunker-version')).toBe(2); + }); + + test('re-put without chunker_version does not lower an already-current version', async () => { + await engine.putPage('dream/preserve-version', { + type: 'concept', + title: 'Preserve me', + compiled_truth: 'first write', + timeline: '', + chunker_version: MARKDOWN_CHUNKER_VERSION, + }); + // Second write omits chunker_version; the UPDATE branch must not drop it + // below the current chunker version. + await engine.putPage('dream/preserve-version', { + type: 'concept', + title: 'Preserve me', + compiled_truth: 'second write', + timeline: '', + }); + expect(await readChunkerVersion('dream/preserve-version')).toBe(MARKDOWN_CHUNKER_VERSION); + }); +});