mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19a291e897 | ||
|
|
85389d37db | ||
|
|
0bbb0bbef8 | ||
|
|
0247c51660 | ||
|
|
a24f437a6f |
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* tasks-41o — one-off, idempotent backfill for `pages.content_created_at`
|
||||
* (migration v125).
|
||||
*
|
||||
* The pglite→Postgres re-ingest (Jul 7-8) hit a latent bug: putPage()'s
|
||||
* INSERT never set created_at (DB default now()) and computeEffectiveDate
|
||||
* never read frontmatter.created, so every re-ingested row's effective_date
|
||||
* fell back to the row-insert timestamp instead of the content's real date.
|
||||
* The fix (migration v125 + effective-date.ts precedence chain) is already
|
||||
* additive-safe; this script is the one-off data repair that makes it visible
|
||||
* on EXISTING rows:
|
||||
*
|
||||
* 1. For every page where `content_created_at IS NULL` and
|
||||
* `frontmatter->>'created'` is present, parse it with the same
|
||||
* `parseDateLoose` the precedence chain uses and write it to
|
||||
* `content_created_at`. NEVER touches `created_at` (row-insert time).
|
||||
* 2. Re-walk `effective_date` / `effective_date_source` for every page via
|
||||
* the existing `backfillEffectiveDate` library (same code path
|
||||
* `gbrain reindex-frontmatter` uses) so the newly-populated
|
||||
* content_created_at immediately wins its precedence slot and search's
|
||||
* effective_date reflects the real content date, not just the column.
|
||||
*
|
||||
* Idempotent: step 1 only touches rows where content_created_at IS NULL, so
|
||||
* a re-run after a partial failure or a later frontmatter fix picks up
|
||||
* exactly the delta. Step 2 reuses backfillEffectiveDate's own
|
||||
* no-op-on-equal guard (skips the UPDATE when the computed value already
|
||||
* matches), so re-running the whole script is always safe and cheap.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/backfill-content-created-at.ts [--dry-run] [--slug-prefix P] [--json]
|
||||
*
|
||||
* --dry-run Count what would change; no DB writes (either step).
|
||||
* --slug-prefix P Scope step 1 to slugs starting with P (e.g. 'wiki/').
|
||||
* Step 2 (effective_date re-walk) always runs over ALL
|
||||
* pages regardless of this flag — it's cheap (no-op-on-equal)
|
||||
* and a partial content_created_at backfill from a prior
|
||||
* run should still get its effective_date corrected.
|
||||
* --json Machine-readable result envelope on stdout.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import { parseDateLoose } from '../src/core/effective-date.ts';
|
||||
import { backfillEffectiveDate } from '../src/core/backfill-effective-date.ts';
|
||||
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
interface CandidateRow {
|
||||
id: number;
|
||||
slug: string;
|
||||
fm_created: string | null;
|
||||
}
|
||||
|
||||
export interface ContentCreatedAtBackfillResult {
|
||||
examined: number;
|
||||
updated: number;
|
||||
skipped_unparseable: number;
|
||||
duration_sec: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Step 1: populate content_created_at from frontmatter->>'created'.
|
||||
* Keyset-paginated (WHERE id > lastId), same shape as backfillEffectiveDate.
|
||||
*/
|
||||
export async function backfillContentCreatedAt(
|
||||
engine: BrainEngine,
|
||||
opts: { dryRun?: boolean; slugPrefix?: string } = {},
|
||||
): Promise<ContentCreatedAtBackfillResult> {
|
||||
const start = Date.now();
|
||||
let lastId = 0;
|
||||
let examined = 0;
|
||||
let updated = 0;
|
||||
let skippedUnparseable = 0;
|
||||
|
||||
const slugPrefix = opts.slugPrefix?.replace(/[\\%_]/g, (c) => '\\' + c) ?? null;
|
||||
|
||||
while (true) {
|
||||
// NOTE: ESCAPE takes a single character. In this TS source that's `'\\'`
|
||||
// (one backslash at runtime) — the doubled `'\\\\'` form sends a 2-char
|
||||
// escape string and Postgres rejects it with "invalid escape string".
|
||||
const slugFilter = slugPrefix ? `AND slug LIKE $2 ESCAPE '\\'` : '';
|
||||
const params: unknown[] = [lastId];
|
||||
if (slugPrefix) params.push(slugPrefix + '%');
|
||||
|
||||
// frontmatter ? 'created': jsonb key-existence operator — cheap pre-filter
|
||||
// before the JS-side parseDateLoose validation. content_created_at IS NULL
|
||||
// is what makes this idempotent: an already-backfilled row never re-matches.
|
||||
const rows = await engine.executeRaw<CandidateRow>(
|
||||
`SELECT id, slug, frontmatter->>'created' AS fm_created
|
||||
FROM pages
|
||||
WHERE id > $1 ${slugFilter}
|
||||
AND content_created_at IS NULL
|
||||
AND frontmatter ? 'created'
|
||||
ORDER BY id
|
||||
LIMIT ${BATCH_SIZE}`,
|
||||
params,
|
||||
);
|
||||
|
||||
if (rows.length === 0) break;
|
||||
examined += rows.length;
|
||||
|
||||
for (const r of rows) {
|
||||
const parsed = parseDateLoose(r.fm_created);
|
||||
if (!parsed) {
|
||||
skippedUnparseable++;
|
||||
continue;
|
||||
}
|
||||
if (!opts.dryRun) {
|
||||
// Guard content_created_at IS NULL again at UPDATE time: belt-and-
|
||||
// suspenders against a concurrent writer (e.g. a live `gbrain sync`)
|
||||
// populating it between the SELECT and this UPDATE.
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET content_created_at = $1::timestamptz WHERE id = $2 AND content_created_at IS NULL`,
|
||||
[parsed.toISOString(), r.id],
|
||||
);
|
||||
}
|
||||
updated++;
|
||||
}
|
||||
|
||||
lastId = rows[rows.length - 1].id;
|
||||
}
|
||||
|
||||
return {
|
||||
examined,
|
||||
updated,
|
||||
skipped_unparseable: skippedUnparseable,
|
||||
duration_sec: (Date.now() - start) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const opts: { dryRun?: boolean; slugPrefix?: string; json?: boolean } = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (a === '--dry-run') opts.dryRun = true;
|
||||
else if (a === '--slug-prefix') opts.slugPrefix = args[++i];
|
||||
else if (a === '--json') opts.json = true;
|
||||
else {
|
||||
console.error(`Unknown arg: ${a}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
const { createEngine } = await import('../src/core/engine-factory.ts');
|
||||
const { loadConfig, toEngineConfig } = await import('../src/core/config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) {
|
||||
console.error('No gbrain config; run `gbrain init` first.');
|
||||
process.exit(1);
|
||||
}
|
||||
const engineConfig = toEngineConfig(cfg);
|
||||
const engine = await createEngine(engineConfig);
|
||||
await engine.connect(engineConfig);
|
||||
// Applies migration v125 (pages.content_created_at) if not already applied.
|
||||
await engine.initSchema();
|
||||
|
||||
try {
|
||||
const step1 = await backfillContentCreatedAt(engine, {
|
||||
dryRun: opts.dryRun,
|
||||
slugPrefix: opts.slugPrefix,
|
||||
});
|
||||
|
||||
// Step 2: re-walk effective_date/effective_date_source for ALL pages so
|
||||
// the newly-populated content_created_at actually changes what search
|
||||
// sees. Cheap: backfillEffectiveDate no-ops any row whose computed value
|
||||
// already matches, so this is safe to run unconditionally (and safe to
|
||||
// re-run this whole script repeatedly).
|
||||
const step2 = await backfillEffectiveDate(engine, { dryRun: opts.dryRun, fresh: true });
|
||||
|
||||
const result = {
|
||||
status: opts.dryRun ? 'dry_run' : 'ok',
|
||||
content_created_at: step1,
|
||||
effective_date_recompute: {
|
||||
examined: step2.examined,
|
||||
updated: step2.updated,
|
||||
fallback: step2.fallback,
|
||||
duration_sec: step2.durationSec,
|
||||
},
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.error(
|
||||
`\ncontent_created_at backfill (${result.status}): ` +
|
||||
`examined=${step1.examined} updated=${step1.updated} skipped_unparseable=${step1.skipped_unparseable} ` +
|
||||
`dur=${step1.duration_sec.toFixed(1)}s\n` +
|
||||
`effective_date recompute: examined=${step2.examined} updated=${step2.updated} ` +
|
||||
`fallback=${step2.fallback} dur=${step2.durationSec.toFixed(1)}s`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if ('disconnect' in engine && typeof engine.disconnect === 'function') {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only run when invoked directly (bun run scripts/backfill-content-created-at.ts),
|
||||
// not when imported by a test.
|
||||
if (import.meta.main) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
+5
-2
@@ -2051,8 +2051,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
// v0.30.1: still works; canonical entrypoint is now `gbrain backfill
|
||||
// effective_date`. This command stays as a thin alias for back-compat.
|
||||
const { reindexFrontmatterCli } = await import('./commands/reindex-frontmatter.ts');
|
||||
await reindexFrontmatterCli(args);
|
||||
return; // reindexFrontmatterCli handles its own engine lifecycle
|
||||
// This dispatcher already owns a connected engine. Passing it through
|
||||
// avoids a second PGLite connection trying to acquire the lock held by
|
||||
// this process.
|
||||
await reindexFrontmatterCli(args, engine);
|
||||
break;
|
||||
}
|
||||
case 'backfill': {
|
||||
// v0.30.1: first-class generic backfill command. Subcommand dispatch
|
||||
|
||||
@@ -252,6 +252,13 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
// tasks-41o: preserve the content's own creation date across engine
|
||||
// migration. Pre-fix, this copy silently dropped it — a pglite→postgres
|
||||
// migration re-INSERTed every row with putPage's default (row-insert
|
||||
// time), the exact bug class that produced the Jul 7-8 timestamp
|
||||
// corruption. `listPages` (both engines) is `SELECT p.*`, so
|
||||
// `page.content_created_at` is already populated when present.
|
||||
content_created_at: page.content_created_at,
|
||||
}, sourceOpts);
|
||||
|
||||
// Copy chunks with embeddings.
|
||||
|
||||
@@ -151,8 +151,14 @@ export async function runReindexFrontmatter(
|
||||
};
|
||||
}
|
||||
|
||||
/** CLI entrypoint. Argv shape matches reindex-code for consistency. */
|
||||
export async function reindexFrontmatterCli(args: string[]): Promise<void> {
|
||||
/**
|
||||
* CLI entrypoint. Argv shape matches reindex-code for consistency.
|
||||
*
|
||||
* When dispatched from cli.ts, `existingEngine` is already connected and is
|
||||
* owned by the caller. Standalone callers may omit it and retain the original
|
||||
* create/connect/init/disconnect lifecycle.
|
||||
*/
|
||||
export async function reindexFrontmatterCli(args: string[], existingEngine?: BrainEngine): Promise<void> {
|
||||
const opts: ReindexFrontmatterOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
@@ -173,21 +179,29 @@ export async function reindexFrontmatterCli(args: string[]): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const { loadConfig, toEngineConfig } = await import('../core/config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) {
|
||||
console.error('No gbrain config; run `gbrain init` first.');
|
||||
process.exit(1);
|
||||
let engine: BrainEngine;
|
||||
let ownsEngine = false;
|
||||
|
||||
if (existingEngine) {
|
||||
engine = existingEngine;
|
||||
} else {
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const { loadConfig, toEngineConfig } = await import('../core/config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) {
|
||||
console.error('No gbrain config; run `gbrain init` first.');
|
||||
process.exit(1);
|
||||
}
|
||||
const engineConfig = toEngineConfig(cfg);
|
||||
engine = await createEngine(engineConfig);
|
||||
// v0.37.7.0 #1225: createEngine() only constructs; callers MUST connect
|
||||
// before any executeRaw call. Pre-fix, the first query in countAffected
|
||||
// crashed with "PGLite not connected. Call connect() first." even on
|
||||
// --dry-run. initSchema is idempotent on a current schema, costs ~1ms.
|
||||
await engine.connect(engineConfig);
|
||||
await engine.initSchema();
|
||||
ownsEngine = true;
|
||||
}
|
||||
const engineConfig = toEngineConfig(cfg);
|
||||
const engine = await createEngine(engineConfig);
|
||||
// v0.37.7.0 #1225: createEngine() only constructs; callers MUST connect
|
||||
// before any executeRaw call. Pre-fix, the first query in countAffected
|
||||
// crashed with "PGLite not connected. Call connect() first." even on
|
||||
// --dry-run. initSchema is idempotent on a current schema, costs ~1ms.
|
||||
await engine.connect(engineConfig);
|
||||
await engine.initSchema();
|
||||
|
||||
try {
|
||||
const result = await runReindexFrontmatter(engine, opts);
|
||||
@@ -202,7 +216,7 @@ export async function reindexFrontmatterCli(args: string[]): Promise<void> {
|
||||
}
|
||||
if (result.status === 'cancelled') process.exit(1);
|
||||
} finally {
|
||||
if ('disconnect' in engine && typeof engine.disconnect === 'function') {
|
||||
if (ownsEngine && 'disconnect' in engine && typeof engine.disconnect === 'function') {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,8 @@ interface PageRow {
|
||||
effective_date_source: EffectiveDateSource | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
/** tasks-41o (migration v125). NULL until scripts/backfill-content-created-at.ts runs. */
|
||||
content_created_at: string | null;
|
||||
}
|
||||
|
||||
function parseFrontmatter(raw: unknown): Record<string, unknown> {
|
||||
@@ -150,8 +152,12 @@ export async function backfillEffectiveDate(
|
||||
|
||||
// Keyset pagination: WHERE id > last_id ORDER BY id LIMIT N. Single-direction
|
||||
// walk; safe under concurrent inserts (new rows show up at the tail).
|
||||
// NOTE: ESCAPE takes a single character. In this TS source that's `'\\'`
|
||||
// (one backslash at runtime) — the doubled `'\\\\'` form sent a 2-char
|
||||
// escape string and every --slug-prefix run died with "invalid escape
|
||||
// string" (fixed in the tasks-41o takeover; PR #3168 review).
|
||||
const slugFilter = slugPrefix
|
||||
? `AND slug LIKE $2 ESCAPE '\\\\'`
|
||||
? `AND slug LIKE $2 ESCAPE '\\'`
|
||||
: '';
|
||||
const params: unknown[] = [lastId];
|
||||
if (slugPrefix) params.push(slugPrefix + '%');
|
||||
@@ -159,7 +165,7 @@ export async function backfillEffectiveDate(
|
||||
const limitParam = `$${params.length}`;
|
||||
|
||||
const rows = await engine.executeRaw<PageRow>(
|
||||
`SELECT id, slug, frontmatter, import_filename, effective_date, effective_date_source, created_at, updated_at
|
||||
`SELECT id, slug, frontmatter, import_filename, effective_date, effective_date_source, created_at, updated_at, content_created_at
|
||||
FROM pages
|
||||
WHERE id > $1 ${slugFilter}
|
||||
ORDER BY id
|
||||
@@ -193,6 +199,11 @@ export async function backfillEffectiveDate(
|
||||
filename,
|
||||
updatedAt: new Date(r.updated_at),
|
||||
createdAt: new Date(r.created_at),
|
||||
// tasks-41o: once scripts/backfill-content-created-at.ts has
|
||||
// populated the column, this re-walk (`gbrain reindex-frontmatter`)
|
||||
// is what actually flips effective_date/effective_date_source for
|
||||
// existing rows — the column ranks just above the 'created' rung.
|
||||
contentCreatedAt: r.content_created_at ? new Date(r.content_created_at) : null,
|
||||
});
|
||||
|
||||
// No-op-on-equal: skip the UPDATE if existing matches (saves write
|
||||
@@ -224,6 +235,7 @@ export async function backfillEffectiveDate(
|
||||
filename,
|
||||
updatedAt: new Date(r.updated_at),
|
||||
createdAt: new Date(r.created_at),
|
||||
contentCreatedAt: r.content_created_at ? new Date(r.content_created_at) : null,
|
||||
});
|
||||
const existingMs = r.effective_date ? new Date(r.effective_date).getTime() : null;
|
||||
const computedMs = computed.date ? computed.date.getTime() : null;
|
||||
|
||||
@@ -23,7 +23,7 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
import { logQualityProbeEvent, readRecentQualityProbeEvents } from '../audit-quality-probe.ts';
|
||||
import { logQualityProbeEvent, readRecentQualityProbeEvents, type QualityProbeAuditEvent, type QualityProbeOutcome } from '../audit-quality-probe.ts';
|
||||
|
||||
/** Run-once gate window in ms. 24h matches the "nightly" cadence. */
|
||||
const NIGHTLY_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
@@ -102,20 +102,38 @@ export async function runNightlyQualityProbe(deps: NightlyProbeDeps): Promise<Ni
|
||||
}
|
||||
|
||||
// 24h rate limit — skip + audit "rate_limited".
|
||||
// Only completed-attempt outcomes count toward the window. Skip rows
|
||||
// (rate_limited) must not refresh it: each skip writes an audit row, so
|
||||
// counting skips would livelock the gate on its own trail (one skip row
|
||||
// per autopilot tick, forever). no_embedding_key DOES count — it's a
|
||||
// completed attempt that shouldn't retry (and re-warn on stderr + re-log)
|
||||
// every 5-min tick on a keyless brain; the key showing up is picked up
|
||||
// within 24h, same cadence as any other outcome.
|
||||
const now = deps.now();
|
||||
const recent = readRecentQualityProbeEvents(2, now); // 2-day window is enough for 24h check
|
||||
const allRecent = readRecentQualityProbeEvents(2, now); // 2-day window is enough for 24h check
|
||||
const runOutcomes: ReadonlySet<QualityProbeOutcome> = new Set(['pass', 'fail', 'inconclusive', 'error', 'budget_exceeded', 'no_embedding_key']);
|
||||
const recent = allRecent.filter((ev) => runOutcomes.has(ev.outcome));
|
||||
const decision = shouldRunNightly(now, recent);
|
||||
if (!decision.run) {
|
||||
logQualityProbeEvent({
|
||||
outcome: 'rate_limited',
|
||||
exit_code: 0,
|
||||
pass_count: 0,
|
||||
fail_count: 0,
|
||||
inconclusive_count: 0,
|
||||
error_count: 0,
|
||||
est_cost_usd: 0,
|
||||
detail: 'already ran within 24h window',
|
||||
});
|
||||
// Log-once per window: skip the duplicate audit row when the latest
|
||||
// row is already rate_limited (the 5-min tick cadence otherwise
|
||||
// writes ~288 identical rows/day and pollutes probe-health).
|
||||
const latest = allRecent.reduce<QualityProbeAuditEvent | null>(
|
||||
(a, b) => (a === null || Date.parse(b.ts) >= Date.parse(a.ts) ? b : a),
|
||||
null,
|
||||
);
|
||||
if (latest === null || latest.outcome !== 'rate_limited') {
|
||||
logQualityProbeEvent({
|
||||
outcome: 'rate_limited',
|
||||
exit_code: 0,
|
||||
pass_count: 0,
|
||||
fail_count: 0,
|
||||
inconclusive_count: 0,
|
||||
error_count: 0,
|
||||
est_cost_usd: 0,
|
||||
detail: 'already ran within 24h window',
|
||||
});
|
||||
}
|
||||
return { outcome: 'rate_limited', exit_code: 0, detail: 'already ran within 24h' };
|
||||
}
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'sync_failures',
|
||||
'sync_freshness',
|
||||
'takes_weight_grid',
|
||||
'timeline_coverage',
|
||||
'unified_multimodal_coverage',
|
||||
'voice_gate_health',
|
||||
]);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* v0.29.1 — Compute a page's effective_date from frontmatter precedence.
|
||||
* tasks-41o — added the content_created_at / frontmatter.created rungs.
|
||||
*
|
||||
* The "effective date" is the answer to "when was this page about?" It's
|
||||
* NOT updated_at (which churns from auto-link) and NOT created_at (which
|
||||
@@ -10,8 +11,29 @@
|
||||
* 2. frontmatter.date — dated essays
|
||||
* 3. frontmatter.published — writing/
|
||||
* 4. filename-date — leading YYYY-MM-DD in basename
|
||||
* 5. updated_at — fallback
|
||||
* 6. created_at — last resort (only if updated_at NULL)
|
||||
* 5. content_created_at — the pages.content_created_at column (see
|
||||
* PageInput.content_created_at). Both
|
||||
* in-tree writers derive it from
|
||||
* frontmatter.created, so it ranks WITH
|
||||
* that signal — just above it, because a
|
||||
* persisted column survives a later
|
||||
* frontmatter edit that drops the key.
|
||||
* Ranking it above event_date/date/
|
||||
* published would let a generic `created`
|
||||
* stamp silently flip effective_date on
|
||||
* every reindex of a page that also
|
||||
* carries a deliberate date field.
|
||||
* 6. frontmatter.created — generic "content created" signal (e.g.
|
||||
* wiki/entities/ pages with no
|
||||
* event_date/date/published/filename
|
||||
* date). Ranked below the dedicated
|
||||
* content-date fields above because those
|
||||
* are deliberate, page-type-specific
|
||||
* signals; `created` is a broader,
|
||||
* weaker one used as a last resort before
|
||||
* falling back to row bookkeeping.
|
||||
* 7. updated_at — fallback
|
||||
* 8. created_at — last resort (only if updated_at NULL)
|
||||
*
|
||||
* Per-prefix override: for `daily/` and `meetings/` slug prefixes, the
|
||||
* filename-date jumps to position 1 — the filename is the user's primary
|
||||
@@ -43,6 +65,13 @@ export interface ComputeEffectiveDateOpts {
|
||||
filename?: string | null;
|
||||
updatedAt: Date;
|
||||
createdAt: Date;
|
||||
/**
|
||||
* tasks-41o: the page's `content_created_at` column value (or an
|
||||
* importer-computed value before the row exists). Ranks just above the
|
||||
* frontmatter.created rung — see the module doc comment. Optional/
|
||||
* undefined for pre-migration callers; treated the same as null.
|
||||
*/
|
||||
contentCreatedAt?: Date | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -118,13 +147,23 @@ function hasFilenameFirstPrefix(slug: string): boolean {
|
||||
* 'fallback' when nothing in frontmatter or filename parses.
|
||||
*/
|
||||
export function computeEffectiveDate(opts: ComputeEffectiveDateOpts): EffectiveDateResult {
|
||||
const { slug, frontmatter, filename, updatedAt, createdAt } = opts;
|
||||
const { slug, frontmatter, filename, updatedAt, createdAt, contentCreatedAt } = opts;
|
||||
const filenameFirst = hasFilenameFirstPrefix(slug);
|
||||
|
||||
// tasks-41o: the persisted pages.content_created_at column. Derived from
|
||||
// frontmatter.created by every in-tree writer, so it ranks WITH that
|
||||
// signal (just above it — the column survives a frontmatter edit that
|
||||
// drops the key). See the module doc comment.
|
||||
const contentCreated = validateInRange(contentCreatedAt ?? null);
|
||||
|
||||
const fmEvent = validateInRange(parseDateLoose(frontmatter.event_date));
|
||||
const fmDate = validateInRange(parseDateLoose(frontmatter.date));
|
||||
const fmPublished = validateInRange(parseDateLoose(frontmatter.published));
|
||||
const filenameDate = extractFilenameDate(filename);
|
||||
// tasks-41o: generic "content created" signal — weaker than the
|
||||
// page-type-specific fields above, so it's ranked last before the
|
||||
// updated_at/created_at fallback tier (see module doc comment).
|
||||
const fmCreated = validateInRange(parseDateLoose(frontmatter.created));
|
||||
|
||||
// Build the ordered candidate list. For filename-first prefixes
|
||||
// (daily/, meetings/) the filename moves to the head of the chain.
|
||||
@@ -134,12 +173,16 @@ export function computeEffectiveDate(opts: ComputeEffectiveDateOpts): EffectiveD
|
||||
{ date: fmEvent, source: 'event_date' },
|
||||
{ date: fmDate, source: 'date' },
|
||||
{ date: fmPublished, source: 'published' },
|
||||
{ date: contentCreated, source: 'content_created_at' },
|
||||
{ date: fmCreated, source: 'created' },
|
||||
]
|
||||
: [
|
||||
{ date: fmEvent, source: 'event_date' },
|
||||
{ date: fmDate, source: 'date' },
|
||||
{ date: fmPublished, source: 'published' },
|
||||
{ date: filenameDate, source: 'filename' },
|
||||
{ date: contentCreated, source: 'content_created_at' },
|
||||
{ date: fmCreated, source: 'created' },
|
||||
];
|
||||
|
||||
for (const c of candidates) {
|
||||
|
||||
+14
-1
@@ -11,7 +11,7 @@ import { extractCodeRefs, imageOfCandidates } from './link-extraction.ts';
|
||||
import { embedBatch, embedMultimodal, currentEmbeddingSignature } from './embedding.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
|
||||
import type { ChunkInput, PageInput, PageType } from './types.ts';
|
||||
import { computeEffectiveDate } from './effective-date.ts';
|
||||
import { computeEffectiveDate, parseDateLoose } from './effective-date.ts';
|
||||
import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts';
|
||||
import { logSlugFallback } from './audit-slug-fallback.ts';
|
||||
import { resolveContextualRetrievalMode } from './contextual-retrieval-resolver.ts';
|
||||
@@ -752,12 +752,24 @@ export async function importFromContent(
|
||||
// consults it.
|
||||
const filenameForChain = opts.filename ?? slug.split('/').pop() ?? slug;
|
||||
const nowDate = new Date();
|
||||
// tasks-41o — thread frontmatter.created forward so it PERSISTS on the
|
||||
// row as content_created_at (distinct from created_at, the row-insert
|
||||
// time; this NEVER touches created_at). Passed as contentCreatedAt too
|
||||
// so import and the reindex re-walk (backfill-effective-date.ts, which
|
||||
// reads the column) compute the same result — the opt ranks just above
|
||||
// the 'created' rung, never above event_date/date/published/filename.
|
||||
// `existing` doesn't carry content_created_at (engine.getPage's SELECT
|
||||
// doesn't project it, same as effective_date); putPage's COALESCE-
|
||||
// preserve UPDATE keeps any prior column value when this frontmatter
|
||||
// has no `created` key.
|
||||
const contentCreatedAt = parseDateLoose(parsed.frontmatter.created);
|
||||
const { date: effectiveDate, source: effectiveDateSource } = computeEffectiveDate({
|
||||
slug,
|
||||
frontmatter: parsed.frontmatter,
|
||||
filename: filenameForChain,
|
||||
updatedAt: existing?.updated_at ?? nowDate,
|
||||
createdAt: existing?.created_at ?? nowDate,
|
||||
contentCreatedAt,
|
||||
});
|
||||
|
||||
await tx.putPage(slug, {
|
||||
@@ -770,6 +782,7 @@ export async function importFromContent(
|
||||
effective_date: effectiveDate,
|
||||
effective_date_source: effectiveDateSource,
|
||||
import_filename: filenameForChain,
|
||||
content_created_at: contentCreatedAt,
|
||||
// v0.32.7 CJK wave: stamp the chunker version so the post-upgrade
|
||||
// reindex sweep can find pre-bump pages via `chunker_version < 2`.
|
||||
// Also capture the repo-relative source path so sync's delete/rename
|
||||
|
||||
@@ -5671,6 +5671,38 @@ export const MIGRATIONS: Migration[] = [
|
||||
`);
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 125,
|
||||
name: 'pages_content_created_at',
|
||||
// tasks-41o — the pglite→Postgres re-ingest (Jul 7-8) exposed a latent
|
||||
// gap: putPage()'s INSERT never set created_at (DB default now()) and
|
||||
// hardcoded updated_at=now(), so every re-ingested row's ROW timestamps
|
||||
// read as "just now" regardless of when the underlying content was
|
||||
// actually authored. created_at is correctly the row-insert time (never
|
||||
// touched by this migration — that IS its semantics); the gap is that
|
||||
// nothing captured the CONTENT's own creation date when frontmatter
|
||||
// supplied one (e.g. `created: 2026-05-14` on a wiki/entities/ page with
|
||||
// no event_date/date/published to feed the existing effective_date
|
||||
// chain). Additive + idempotent, modeled on the PR #2533 / migration
|
||||
// v121 (facts.dimension) precedent: a single nullable column, no
|
||||
// backfill inside the migration itself (see scripts/backfill-content-
|
||||
// created-at.ts — a separate, re-runnable, idempotent pass so a bad
|
||||
// backfill can be re-run without a schema round-trip). Consulted by
|
||||
// computeEffectiveDate (src/core/effective-date.ts) just above the
|
||||
// frontmatter.created rung, and settable via
|
||||
// PageInput.content_created_at on putPage. Mirrored in src/schema.sql,
|
||||
// src/core/pglite-schema.ts, and the generated src/core/schema-embedded.ts
|
||||
// so fresh installs carry the same column.
|
||||
//
|
||||
// Renumbered to v125 on rebase: master claimed v123 (configurable_fts_language,
|
||||
// #2941) and v124 (page_search_vector_drop_compiled_truth, #2704). Taking the
|
||||
// next free slot avoids schema-version drift on brains already at v124.
|
||||
idempotent: true,
|
||||
sql: `
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS content_created_at TIMESTAMPTZ;
|
||||
`,
|
||||
|
||||
},
|
||||
];
|
||||
|
||||
export const LATEST_VERSION = MIGRATIONS.length > 0
|
||||
|
||||
@@ -1023,6 +1023,11 @@ export class PGLiteEngine implements BrainEngine {
|
||||
: (page.effective_date ?? null);
|
||||
const effectiveDateSource = page.effective_date_source ?? null;
|
||||
const importFilename = page.import_filename ?? null;
|
||||
// tasks-41o (migration v125) — content_created_at mirrors the
|
||||
// effective_date COALESCE-preserve shape. Mirrors postgres-engine.ts.
|
||||
const contentCreatedAt = page.content_created_at instanceof Date
|
||||
? page.content_created_at.toISOString()
|
||||
: (page.content_created_at ?? null);
|
||||
// v0.32.7 CJK wave: chunker_version + source_path columns.
|
||||
const chunkerVersion = page.chunker_version ?? null;
|
||||
const sourcePath = page.source_path ?? null;
|
||||
@@ -1035,8 +1040,8 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const ingestedVia = page.ingested_via ?? null;
|
||||
const ingestedAt = (sourceKind || sourceUri || ingestedVia) ? new Date().toISOString() : null;
|
||||
const { rows } = await this.db.query(
|
||||
`INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12, COALESCE($13, ${MARKDOWN_CHUNKER_VERSION}), $14, $15, $16, $17, $18::timestamptz)
|
||||
`INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, content_created_at, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12, $13::timestamptz, COALESCE($14, ${MARKDOWN_CHUNKER_VERSION}), $15, $16, $17, $18, $19::timestamptz)
|
||||
ON CONFLICT (source_id, slug) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
page_kind = EXCLUDED.page_kind,
|
||||
@@ -1049,14 +1054,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
effective_date = COALESCE(EXCLUDED.effective_date, pages.effective_date),
|
||||
effective_date_source = COALESCE(EXCLUDED.effective_date_source, pages.effective_date_source),
|
||||
import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename),
|
||||
content_created_at = COALESCE(EXCLUDED.content_created_at, pages.content_created_at),
|
||||
chunker_version = COALESCE(EXCLUDED.chunker_version, pages.chunker_version),
|
||||
source_path = COALESCE(EXCLUDED.source_path, pages.source_path),
|
||||
source_kind = COALESCE(EXCLUDED.source_kind, pages.source_kind),
|
||||
source_uri = COALESCE(EXCLUDED.source_uri, pages.source_uri),
|
||||
ingested_via = COALESCE(EXCLUDED.ingested_via, pages.ingested_via),
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at)
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`,
|
||||
[sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, content_created_at, source_kind, source_uri, ingested_via, ingested_at`,
|
||||
[sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, contentCreatedAt, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
|
||||
);
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
@@ -96,6 +96,10 @@ CREATE TABLE IF NOT EXISTS pages (
|
||||
effective_date_source TEXT,
|
||||
import_filename TEXT,
|
||||
salience_touched_at TIMESTAMPTZ,
|
||||
-- tasks-41o (migration v125): the content's own creation date -- distinct
|
||||
-- from created_at (the row-insert time; never overwritten). Ranks just above
|
||||
-- the frontmatter.created rung in computeEffectiveDate. Mirrors src/schema.sql.
|
||||
content_created_at TIMESTAMPTZ,
|
||||
-- v0.37.0 (migration v79): real stale-page signal for gbrain lsd
|
||||
-- (mirrors src/schema.sql). NULL = never retrieved.
|
||||
last_retrieved_at TIMESTAMPTZ,
|
||||
|
||||
@@ -1084,6 +1084,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
const effectiveDate = page.effective_date ?? null;
|
||||
const effectiveDateSource = page.effective_date_source ?? null;
|
||||
const importFilename = page.import_filename ?? null;
|
||||
// tasks-41o — content_created_at is the content's own creation date
|
||||
// (distinct from created_at, the row-insert time — never touched here).
|
||||
// Same COALESCE-preserve shape as effective_date: omitting it on a
|
||||
// later putPage (auto-link, code reindex, etc.) doesn't blank it out.
|
||||
const contentCreatedAt = page.content_created_at ?? null;
|
||||
// v0.32.7 CJK wave: chunker_version + source_path columns.
|
||||
const chunkerVersion = page.chunker_version ?? null;
|
||||
const sourcePath = page.source_path ?? null;
|
||||
@@ -1097,8 +1102,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
const ingestedVia = page.ingested_via ?? null;
|
||||
const ingestedAt = (sourceKind || sourceUri || ingestedVia) ? new Date() : null;
|
||||
const rows = await sql`
|
||||
INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at)
|
||||
VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters<typeof sql.json>[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}, COALESCE(${chunkerVersion}::smallint, ${MARKDOWN_CHUNKER_VERSION}), ${sourcePath}, ${sourceKind}, ${sourceUri}, ${ingestedVia}, ${ingestedAt})
|
||||
INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, content_created_at, chunker_version, source_path, source_kind, source_uri, ingested_via, ingested_at)
|
||||
VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters<typeof sql.json>[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}, ${contentCreatedAt}, COALESCE(${chunkerVersion}::smallint, ${MARKDOWN_CHUNKER_VERSION}), ${sourcePath}, ${sourceKind}, ${sourceUri}, ${ingestedVia}, ${ingestedAt})
|
||||
ON CONFLICT (source_id, slug) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
page_kind = EXCLUDED.page_kind,
|
||||
@@ -1111,13 +1116,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
effective_date = COALESCE(EXCLUDED.effective_date, pages.effective_date),
|
||||
effective_date_source = COALESCE(EXCLUDED.effective_date_source, pages.effective_date_source),
|
||||
import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename),
|
||||
content_created_at = COALESCE(EXCLUDED.content_created_at, pages.content_created_at),
|
||||
chunker_version = COALESCE(EXCLUDED.chunker_version, pages.chunker_version),
|
||||
source_path = COALESCE(EXCLUDED.source_path, pages.source_path),
|
||||
source_kind = COALESCE(EXCLUDED.source_kind, pages.source_kind),
|
||||
source_uri = COALESCE(EXCLUDED.source_uri, pages.source_uri),
|
||||
ingested_via = COALESCE(EXCLUDED.ingested_via, pages.ingested_via),
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at)
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, content_created_at, source_kind, source_uri, ingested_via, ingested_at
|
||||
`;
|
||||
return rowToPage(rows[0]);
|
||||
}
|
||||
|
||||
@@ -123,6 +123,12 @@ CREATE TABLE IF NOT EXISTS pages (
|
||||
effective_date_source TEXT,
|
||||
import_filename TEXT,
|
||||
salience_touched_at TIMESTAMPTZ,
|
||||
-- tasks-41o (migration v125): the content's own creation date, distinct
|
||||
-- from created_at (row-insert time, never overwritten). NULL unless
|
||||
-- explicitly supplied (frontmatter \`created\`, backfilled from history, or
|
||||
-- caller override via PageInput.content_created_at). Ranks just above the
|
||||
-- frontmatter.created rung in computeEffectiveDate's precedence chain.
|
||||
content_created_at TIMESTAMPTZ,
|
||||
-- v0.37.0 (migration v79): real stale-page signal for \`gbrain lsd\`. Bumped
|
||||
-- by op-layer write-back inside \`search\`/\`query\`/\`get_page\` op handlers
|
||||
-- (NOT inside engine methods — internal callers must not pollute the
|
||||
|
||||
@@ -123,6 +123,15 @@ export interface Page {
|
||||
* imported pre-v0.29.1.
|
||||
*/
|
||||
import_filename?: string | null;
|
||||
/**
|
||||
* tasks-41o (migration v125): the content's own creation date — distinct
|
||||
* from `created_at` (row-insert time, never overwritten by this field).
|
||||
* NULL unless explicitly supplied: frontmatter `created`, a backfill from
|
||||
* git/mtime history, or an importer/caller override. Ranks just above the
|
||||
* frontmatter.created rung in `computeEffectiveDate`'s precedence chain —
|
||||
* see `src/core/effective-date.ts`.
|
||||
*/
|
||||
content_created_at?: Date | null;
|
||||
/**
|
||||
* v0.29.1: bumped by `recompute_emotional_weight` when the page's
|
||||
* emotional_weight changes. The salience query window uses
|
||||
@@ -178,9 +187,11 @@ export interface Page {
|
||||
}
|
||||
|
||||
export type EffectiveDateSource =
|
||||
| 'content_created_at'
|
||||
| 'event_date'
|
||||
| 'date'
|
||||
| 'published'
|
||||
| 'created'
|
||||
| 'filename'
|
||||
| 'fallback';
|
||||
|
||||
@@ -233,6 +244,16 @@ export interface PageInput {
|
||||
effective_date_source?: EffectiveDateSource | null;
|
||||
/** v0.29.1: basename without extension captured at import. */
|
||||
import_filename?: string | null;
|
||||
/**
|
||||
* tasks-41o (migration v125): the content's own creation date, distinct
|
||||
* from `created_at` (row-insert time — putPage never derives this from
|
||||
* `created_at` and never lets this field overwrite `created_at`). When
|
||||
* omitted, putPage leaves the column unchanged on conflict (preserves any
|
||||
* existing value, same COALESCE-preserve shape as effective_date); on
|
||||
* insert the column is NULL. Consulted by `computeEffectiveDate` just
|
||||
* above the frontmatter.created rung.
|
||||
*/
|
||||
content_created_at?: Date | null;
|
||||
/**
|
||||
* v0.32.7 CJK wave: bumped to MARKDOWN_CHUNKER_VERSION (2) on import so the
|
||||
* post-upgrade `gbrain reindex --markdown` sweep can find pre-bump pages
|
||||
|
||||
@@ -102,6 +102,8 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
const salienceTouchedAt = readOptionalDate(row.salience_touched_at);
|
||||
const effectiveDateSource = row.effective_date_source as Page['effective_date_source'] | undefined;
|
||||
const importFilename = row.import_filename as string | null | undefined;
|
||||
// tasks-41o (migration v125): three-state read, same shape as effective_date.
|
||||
const contentCreatedAt = readOptionalDate(row.content_created_at);
|
||||
// v0.39.3.0 CV5 — three-state read for provenance columns. Matches the
|
||||
// v0.26.5 deleted_at pattern: undefined when the SELECT projection didn't
|
||||
// include the column (older code paths); null when the column is NULL
|
||||
@@ -128,6 +130,7 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
...(effectiveDate !== undefined && { effective_date: effectiveDate }),
|
||||
...(effectiveDateSource !== undefined && { effective_date_source: effectiveDateSource }),
|
||||
...(importFilename !== undefined && { import_filename: importFilename }),
|
||||
...(contentCreatedAt !== undefined && { content_created_at: contentCreatedAt }),
|
||||
...(salienceTouchedAt !== undefined && { salience_touched_at: salienceTouchedAt }),
|
||||
// v0.39.3.0 (columns added in migration v81 — WARN-8 + CV5). Three-state
|
||||
// optional read; absent SELECT projections compile unchanged.
|
||||
|
||||
@@ -119,6 +119,12 @@ CREATE TABLE IF NOT EXISTS pages (
|
||||
effective_date_source TEXT,
|
||||
import_filename TEXT,
|
||||
salience_touched_at TIMESTAMPTZ,
|
||||
-- tasks-41o (migration v125): the content's own creation date, distinct
|
||||
-- from created_at (row-insert time, never overwritten). NULL unless
|
||||
-- explicitly supplied (frontmatter `created`, backfilled from history, or
|
||||
-- caller override via PageInput.content_created_at). Ranks just above the
|
||||
-- frontmatter.created rung in computeEffectiveDate's precedence chain.
|
||||
content_created_at TIMESTAMPTZ,
|
||||
-- v0.37.0 (migration v79): real stale-page signal for `gbrain lsd`. Bumped
|
||||
-- by op-layer write-back inside `search`/`query`/`get_page` op handlers
|
||||
-- (NOT inside engine methods — internal callers must not pollute the
|
||||
|
||||
@@ -124,6 +124,10 @@ describe('categorizeCheck', () => {
|
||||
expect(categorizeCheck('sync_freshness')).toBe('brain');
|
||||
});
|
||||
|
||||
test('timeline_coverage (emitted by onboard/checks.ts, not doctor.ts) is brain, not meta fallthrough (#2817)', () => {
|
||||
expect(categorizeCheck('timeline_coverage')).toBe('brain');
|
||||
});
|
||||
|
||||
test('returns the right category for a known skill name', () => {
|
||||
expect(categorizeCheck('resolver_health')).toBe('skill');
|
||||
expect(categorizeCheck('skill_conformance')).toBe('skill');
|
||||
|
||||
@@ -17,6 +17,7 @@ function run(opts: {
|
||||
filename?: string | null;
|
||||
updatedAt?: Date;
|
||||
createdAt?: Date;
|
||||
contentCreatedAt?: Date | null;
|
||||
}) {
|
||||
return computeEffectiveDate({
|
||||
slug: opts.slug ?? 'wiki/example',
|
||||
@@ -24,6 +25,7 @@ function run(opts: {
|
||||
filename: opts.filename ?? null,
|
||||
updatedAt: opts.updatedAt ?? baseUpdated,
|
||||
createdAt: opts.createdAt ?? baseCreated,
|
||||
contentCreatedAt: opts.contentCreatedAt ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -168,3 +170,101 @@ describe('computeEffectiveDate range validation [1990, NOW + 1y]', () => {
|
||||
expect(r.source).toBe('fallback');
|
||||
});
|
||||
});
|
||||
|
||||
// tasks-41o — pglite→Postgres re-ingest exposed a latent gap: rows got
|
||||
// row-insert timestamps (created_at/updated_at = now()) instead of their
|
||||
// real content date. content_created_at (the explicit override) and
|
||||
// frontmatter.created (the generic signal) close it.
|
||||
describe('computeEffectiveDate content_created_at / frontmatter.created (tasks-41o)', () => {
|
||||
test('content_created_at loses to deliberate date fields (event_date) — created-derived, must not flip them on reindex', () => {
|
||||
const r = run({
|
||||
fm: { event_date: '2024-04-01' },
|
||||
contentCreatedAt: new Date('2026-05-14T00:00:00Z'),
|
||||
});
|
||||
expect(r.source).toBe('event_date');
|
||||
expect(r.date?.toISOString().startsWith('2024-04-01')).toBe(true);
|
||||
});
|
||||
|
||||
test('content_created_at loses to daily/meetings filename-first override', () => {
|
||||
const r = run({
|
||||
slug: 'meetings/2024-06-15-acme-call',
|
||||
filename: '2024-06-15-acme-call',
|
||||
contentCreatedAt: new Date('2026-05-14T00:00:00Z'),
|
||||
});
|
||||
expect(r.source).toBe('filename');
|
||||
});
|
||||
|
||||
test('content_created_at wins over frontmatter.created and the updated_at/created_at fallback', () => {
|
||||
const r = run({
|
||||
fm: { created: '2024-04-01' },
|
||||
contentCreatedAt: new Date('2026-05-14T00:00:00Z'),
|
||||
});
|
||||
expect(r.source).toBe('content_created_at');
|
||||
expect(r.date?.toISOString().startsWith('2026-05-14')).toBe(true);
|
||||
});
|
||||
|
||||
test('import and reindex agree: created + date page keeps source=date whether or not the column is populated', () => {
|
||||
// Regression for the reindex-flip bug: import computed 'date' while the
|
||||
// re-walk (which reads the persisted column) computed content_created_at
|
||||
// as top priority, silently flipping effective_date on the first
|
||||
// `gbrain reindex-frontmatter` run.
|
||||
const withoutColumn = run({ fm: { created: '2026-01-01', date: '2024-04-01' } });
|
||||
const withColumn = run({
|
||||
fm: { created: '2026-01-01', date: '2024-04-01' },
|
||||
contentCreatedAt: new Date('2026-01-01T00:00:00Z'),
|
||||
});
|
||||
expect(withoutColumn.source).toBe('date');
|
||||
expect(withColumn.source).toBe('date');
|
||||
expect(withColumn.date?.getTime()).toBe(withoutColumn.date?.getTime());
|
||||
});
|
||||
|
||||
test('frontmatter.created wins when no event_date/date/published/filename present (entity-page case)', () => {
|
||||
const r = run({
|
||||
slug: 'wiki/entities/alice-and-charlie-example',
|
||||
fm: { created: '2026-05-14' },
|
||||
});
|
||||
expect(r.source).toBe('created');
|
||||
expect(r.date?.toISOString().startsWith('2026-05-14')).toBe(true);
|
||||
});
|
||||
|
||||
test('frontmatter.created loses to event_date/date/published/filename (weakest content signal)', () => {
|
||||
const r = run({ fm: { created: '2026-01-01', date: '2024-04-01' } });
|
||||
expect(r.source).toBe('date');
|
||||
});
|
||||
|
||||
test('frontmatter.created loses to filename date too', () => {
|
||||
const r = run({ fm: { created: '2026-01-01' }, filename: '2024-06-15-something' });
|
||||
expect(r.source).toBe('filename');
|
||||
});
|
||||
|
||||
test('frontmatter.created still beats the updated_at/created_at fallback', () => {
|
||||
const r = run({ fm: { created: '2026-05-14' } });
|
||||
expect(r.source).toBe('created');
|
||||
expect(r.date?.toISOString().startsWith('2026-05-14')).toBe(true);
|
||||
});
|
||||
|
||||
test('unparseable frontmatter.created falls through to fallback', () => {
|
||||
const r = run({ fm: { created: 'garbage' } });
|
||||
expect(r.source).toBe('fallback');
|
||||
});
|
||||
|
||||
test('out-of-range content_created_at drops to next candidate', () => {
|
||||
const r = run({
|
||||
fm: { date: '2024-04-01' },
|
||||
contentCreatedAt: new Date('1900-01-01T00:00:00Z'),
|
||||
});
|
||||
expect(r.source).toBe('date');
|
||||
});
|
||||
|
||||
test('content_created_at undefined (pre-migration caller) behaves like existing chain', () => {
|
||||
const r = computeEffectiveDate({
|
||||
slug: 'wiki/example',
|
||||
frontmatter: { date: '2024-04-01' },
|
||||
filename: null,
|
||||
updatedAt: baseUpdated,
|
||||
createdAt: baseCreated,
|
||||
// contentCreatedAt omitted entirely
|
||||
});
|
||||
expect(r.source).toBe('date');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
type NightlyProbeResult,
|
||||
} from '../src/core/cycle/nightly-quality-probe.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { computeQualityProbeAuditFilename } from '../src/core/audit-quality-probe.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hermetic audit dir per test
|
||||
@@ -146,6 +147,56 @@ describe('runNightlyQualityProbe (DI stub harness)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('REGRESSION: a history of only rate_limited rows must not block the run (livelock guard)', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
|
||||
// Seed the current-week audit file with skip rows ONLY — the exact
|
||||
// state the 5-min tick cadence produces after one stale real run.
|
||||
const nowMs = Date.now();
|
||||
const rows: string[] = [];
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
rows.push(JSON.stringify({
|
||||
ts: new Date(nowMs - i * 5 * 60000).toISOString(),
|
||||
outcome: 'rate_limited', exit_code: 0, pass_count: 0, fail_count: 0,
|
||||
inconclusive_count: 0, error_count: 0, est_cost_usd: 0,
|
||||
detail: 'already ran within 24h window',
|
||||
}));
|
||||
}
|
||||
writeFileSync(join(auditTmp, computeQualityProbeAuditFilename(new Date())), rows.join('\n') + '\n');
|
||||
const r = await runNightlyQualityProbe(makeDeps());
|
||||
expect(r.outcome).toBe('pass'); // gate ignored the skip rows and ran
|
||||
});
|
||||
});
|
||||
|
||||
test('REGRESSION: rate_limited audit row is written once per window, not per tick', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
|
||||
await runNightlyQualityProbe(makeDeps()); // real run -> pass row
|
||||
const r2 = await runNightlyQualityProbe(makeDeps()); // blocked -> writes ONE rate_limited row
|
||||
const r3 = await runNightlyQualityProbe(makeDeps()); // blocked again -> must NOT write another
|
||||
expect(r2.outcome).toBe('rate_limited');
|
||||
expect(r3.outcome).toBe('rate_limited');
|
||||
const events = await readEvents();
|
||||
expect(events.length).toBe(2);
|
||||
expect(events[1].outcome).toBe('rate_limited');
|
||||
});
|
||||
});
|
||||
|
||||
test('REGRESSION: no_embedding_key counts as a completed attempt — no re-warn/re-log per tick on a keyless brain', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
|
||||
const keyless = () => makeDeps({ hasEmbeddingProvider: async () => false });
|
||||
const r1 = await runNightlyQualityProbe(keyless());
|
||||
expect(r1.outcome).toBe('no_embedding_key'); // one warn + one audit row
|
||||
const r2 = await runNightlyQualityProbe(keyless());
|
||||
expect(r2.outcome).toBe('rate_limited'); // gated, not another no_embedding_key
|
||||
const r3 = await runNightlyQualityProbe(keyless());
|
||||
expect(r3.outcome).toBe('rate_limited');
|
||||
const events = await readEvents();
|
||||
// Exactly 2 rows: the no_embedding_key attempt + ONE rate_limited marker.
|
||||
expect(events.length).toBe(2);
|
||||
expect(events[0].outcome).toBe('no_embedding_key');
|
||||
expect(events[1].outcome).toBe('rate_limited');
|
||||
});
|
||||
});
|
||||
|
||||
test('enabled + PASS summary → outcome: pass with audit row', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
|
||||
const r = await runNightlyQualityProbe(makeDeps());
|
||||
|
||||
@@ -88,6 +88,39 @@ describe('PGLiteEngine: Pages', () => {
|
||||
expect(matches.length).toBe(1);
|
||||
});
|
||||
|
||||
// tasks-41o — content_created_at is a new putPage input (migration v125).
|
||||
// getPage's SELECT deliberately doesn't project it (matches the existing
|
||||
// effective_date/import_filename gap on both engines), so these tests read
|
||||
// it back via listPages (`SELECT p.*`), same as migrate-engine.ts does.
|
||||
test('putPage persists content_created_at (round trip via listPages)', async () => {
|
||||
const contentCreatedAt = new Date('2026-05-14T00:00:00Z');
|
||||
await engine.putPage('test/content-created-at', { ...testPage, content_created_at: contentCreatedAt });
|
||||
|
||||
const rows = await engine.listPages({ slugPrefix: 'test/content-created-at', limit: 10 });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].content_created_at?.toISOString()).toBe(contentCreatedAt.toISOString());
|
||||
});
|
||||
|
||||
test('putPage without content_created_at leaves the column NULL on insert', async () => {
|
||||
await engine.putPage('test/no-content-created-at', testPage);
|
||||
const rows = await engine.listPages({ slugPrefix: 'test/no-content-created-at', limit: 10 });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].content_created_at ?? null).toBeNull();
|
||||
});
|
||||
|
||||
test('putPage COALESCE-preserves content_created_at across a re-put that omits it', async () => {
|
||||
const contentCreatedAt = new Date('2026-05-14T00:00:00Z');
|
||||
await engine.putPage('test/preserve-content-created-at', { ...testPage, content_created_at: contentCreatedAt });
|
||||
// Re-put without content_created_at — must NOT blank out the prior value
|
||||
// (matches the effective_date / import_filename COALESCE-preserve shape).
|
||||
await engine.putPage('test/preserve-content-created-at', { ...testPage, title: 'Retitled' });
|
||||
|
||||
const rows = await engine.listPages({ slugPrefix: 'test/preserve-content-created-at', limit: 10 });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].title).toBe('Retitled');
|
||||
expect(rows[0].content_created_at?.toISOString()).toBe(contentCreatedAt.toISOString());
|
||||
});
|
||||
|
||||
test('getPage returns null for missing slug', async () => {
|
||||
const result = await engine.getPage('nonexistent/slug');
|
||||
expect(result).toBeNull();
|
||||
|
||||
@@ -10,9 +10,9 @@
|
||||
* happy path without throwing.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, mock } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runReindexFrontmatter } from '../src/commands/reindex-frontmatter.ts';
|
||||
import { reindexFrontmatterCli, runReindexFrontmatter } from '../src/commands/reindex-frontmatter.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
@@ -61,4 +61,36 @@ describe('reindex-frontmatter connect-before-query (#1225)', () => {
|
||||
expect(result.status).toBe('dry_run');
|
||||
expect(result.examined).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
test('reuses the CLI-owned engine instead of constructing a second PGLite connection', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO pages (slug, type, title, compiled_truth, page_kind, frontmatter, effective_date)
|
||||
VALUES ($1, 'note', $2, $3, 'markdown', $4::jsonb, NULL)`,
|
||||
[
|
||||
'cli-owned-engine',
|
||||
'CLI owned engine',
|
||||
'# CLI owned engine\n\nbody',
|
||||
JSON.stringify({ effective_date: '2026-01-01' }),
|
||||
],
|
||||
);
|
||||
|
||||
const disconnect = engine.disconnect.bind(engine);
|
||||
const disconnectSpy = mock(disconnect);
|
||||
engine.disconnect = disconnectSpy;
|
||||
const logs: string[] = [];
|
||||
const originalLog = console.log;
|
||||
console.log = (message?: unknown) => logs.push(String(message));
|
||||
|
||||
try {
|
||||
await reindexFrontmatterCli(['--dry-run', '--json'], engine);
|
||||
const result = JSON.parse(logs.at(-1) ?? '{}') as { examined?: number };
|
||||
// A second engine would see a different/empty database; this asserts that
|
||||
// the CLI command queried the connected engine supplied by its dispatcher.
|
||||
expect(result.examined).toBeGreaterThanOrEqual(1);
|
||||
expect(disconnectSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
engine.disconnect = disconnect;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user