fix(extract): derive timeline entries from real-provenance effective dates (takeover of #2877)

DB timeline extraction (extract timeline --source db and extract --stale)
now emits one deterministic fallback timeline row for pages with no
explicit markdown timeline row, anchored on pages.effective_date — but
ONLY when the date has real provenance (frontmatter event_date/date/
published or a dated filename). The mtime-derived 'fallback' provenance
is skipped: computeEffectiveDate stamps it on nearly every imported page,
so emitting a row for it would create one noise entry per timeline-less
page brain-wide on the first extract run after upgrade.

Rebased #2877 onto master (resolved getPage/listStalePagesForExtraction
conflicts against the RLS scoped-read-transaction refactor) and added
the provenance filter plus regression tests for the fallback/null
provenance skip.

Co-authored-by: ShintaroKawakami <ShintaroKawakami@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

skills/maintain/SKILL.md

@@ -262,8 +262,13 @@ Populate them periodically or after major imports:
   `advises`, `mentions`, `source`). Idempotent. Use `--source fs --dir <brain>`
   if you have a markdown checkout to walk instead.
 - `gbrain extract timeline --source db` — backfill structured timeline entries.
-  Parses `- **YYYY-MM-DD** | summary` lines from page content. Idempotent (DB
-  UNIQUE constraint).
+  Parses `- **YYYY-MM-DD** | summary` lines from page content. When a DB page
+  has no explicit timeline row but does have a real-provenance
+  `pages.effective_date` (frontmatter event_date/date/published or a dated
+  filename — mtime-derived dates are skipped), extraction creates one
+  provenance-tagged fallback entry from that date and the page title.
+  Idempotent (DB UNIQUE constraint). `gbrain extract --stale` applies the
+  same fallback for later page edits.
 - `gbrain extract all --source db` — both in one run.
 - `gbrain graph-query <slug> --depth 2` — verify connectivity (use any well-known
   entity slug as a probe).

src/commands/extract.ts

@@ -39,7 +39,7 @@ import {
   extractFrontmatterLinks, isGlobalBasenameEnabled, LINK_EXTRACTOR_VERSION_TS,
   WIKILINK_BASENAME_LINK_TYPE,
   buildBasenameIndex, queryBasenameIndex, stripCodeBlocks,
-  type UnresolvedFrontmatterRef, type LinkCandidate,
+  type UnresolvedFrontmatterRef, type LinkCandidate, type TimelineCandidate,
 } from '../core/link-extraction.ts';
 import { createProgress } from '../core/progress.ts';
 import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
@@ -529,6 +529,57 @@ export function extractTimelineFromContent(content: string, slug: string): Extra
   return entries;
 }
 
+interface EffectiveDateTimelinePage {
+  slug: string;
+  title: string;
+  effective_date?: Date | string | null;
+  effective_date_source?: string | null;
+}
+
+/**
+ * Effective-date provenances that reflect a REAL date on the page
+ * (frontmatter event_date/date/published, or a dated filename). The
+ * 'fallback' provenance is mtime-derived (updated_at/created_at) —
+ * computeEffectiveDate stamps it on nearly every imported page, so emitting
+ * a timeline row for it would create one noise entry per timeline-less page
+ * brain-wide on the next extract run.
+ */
+const REAL_DATE_PROVENANCE = new Set(['event_date', 'date', 'published', 'filename']);
+
+/**
+ * Return explicit markdown timeline rows, or one deterministic page-level
+ * fallback when the page has a real-provenance effective_date but no
+ * explicit row.
+ *
+ * The fallback is intentionally only used for otherwise-empty pages. This
+ * gives every dated page a structured timeline anchor without duplicating a
+ * richer event already present in markdown. The source key makes the row
+ * provenance visible and keeps repeated extraction idempotent.
+ */
+export function extractTimelineEntriesForPage(
+  content: string,
+  page: EffectiveDateTimelinePage,
+): Array<TimelineCandidate & { source?: string }> {
+  const explicit = parseTimelineEntries(content);
+  if (explicit.length > 0) return explicit;
+  if (!page.effective_date) return [];
+  // Skip mtime-derived ('fallback') or missing provenance — only a date the
+  // author actually put on the page earns a structured timeline anchor.
+  if (!REAL_DATE_PROVENANCE.has(page.effective_date_source ?? '')) return [];
+
+  const effectiveDate = page.effective_date instanceof Date
+    ? page.effective_date
+    : new Date(page.effective_date);
+  if (!Number.isFinite(effectiveDate.getTime())) return [];
+
+  return [{
+    date: effectiveDate.toISOString().slice(0, 10),
+    source: `page.effective_date:${page.effective_date_source}`,
+    summary: page.title.trim() || page.slug,
+    detail: '',
+  }];
+}
+
 // --- Main command ---
 
 export interface ExtractOpts {
@@ -1600,7 +1651,7 @@ async function extractTimelineFromDB(
     }
 
     const fullContent = page.compiled_truth + '\n' + page.timeline;
-    const entries = parseTimelineEntries(fullContent);
+    const entries = extractTimelineEntriesForPage(fullContent, page);
 
     for (const entry of entries) {
       if (dryRunSeen) {
@@ -1610,7 +1661,8 @@ async function extractTimelineFromDB(
         if (jsonMode) {
           process.stdout.write(JSON.stringify({
             action: 'add_timeline', slug, source_id, date: entry.date,
-            summary: entry.summary, ...(entry.detail ? { detail: entry.detail } : {}),
+            summary: entry.summary, ...(entry.source ? { source: entry.source } : {}),
+            ...(entry.detail ? { detail: entry.detail } : {}),
           }) + '\n');
         } else {
           console.log(`  ${slug}: ${entry.date} — ${entry.summary}`);
@@ -1619,7 +1671,7 @@ async function extractTimelineFromDB(
       } else {
         // v0.32.8 F4: thread source_id so the JOIN matches the right page
         // when two sources share the same slug.
-        batch.push({ slug, date: entry.date, summary: entry.summary, detail: entry.detail || '', source_id });
+        batch.push({ slug, date: entry.date, source: entry.source, summary: entry.summary, detail: entry.detail || '', source_id });
         if (batch.length >= BATCH_SIZE) await flush();
       }
     }
@@ -1730,8 +1782,8 @@ async function extractStaleFromDB(
           to_source_id: r.toSourceId, origin_source_id: page.source_id,
         });
       }
-      for (const entry of parseTimelineEntries(fullContent)) {
-        timelineRows.push({ slug: page.slug, date: entry.date, summary: entry.summary, detail: entry.detail || '', source_id: page.source_id });
+      for (const entry of extractTimelineEntriesForPage(fullContent, page)) {
+        timelineRows.push({ slug: page.slug, date: entry.date, source: entry.source, summary: entry.summary, detail: entry.detail || '', source_id: page.source_id });
       }
       // EVERY processed page is stamped (incl. zero-link pages). D4 race fix:
       // stamp with the row's READ updated_at, NOT now() — a concurrent edit

src/core/pglite-engine.ts

@@ -974,6 +974,7 @@ export class PGLiteEngine implements BrainEngine {
     }
     const { rows } = await this.db.query(
       `SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
+              effective_date, effective_date_source, import_filename,
               source_kind, source_uri, ingested_via, ingested_at
        FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
       params
@@ -2628,7 +2629,8 @@ export class PGLiteEngine implements BrainEngine {
     const { rows } = await this.db.query(
       // #1768: engine parity — project the same deterministic full-µs UTC string
       // as postgres-engine.ts so extractStaleFromDB stamps the exact updated_at.
-      `SELECT id, slug, source_id, type, title, compiled_truth, timeline, frontmatter, updated_at,
+      `SELECT id, slug, source_id, type, title, compiled_truth, timeline, frontmatter,
+              effective_date, effective_date_source, updated_at,
               to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS updated_at_iso
          FROM pages
          WHERE ${where}${afterClause}

src/core/postgres-engine.ts

@@ -1026,6 +1026,7 @@ export class PostgresEngine implements BrainEngine {
       const deletedCondition = includeDeleted ? tx`` : tx`AND deleted_at IS NULL`;
       const rows = await tx`
         SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
+               effective_date, effective_date_source, import_filename,
                source_kind, source_uri, ingested_via, ingested_at
         FROM pages
         WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
@@ -2782,7 +2783,8 @@ export class PostgresEngine implements BrainEngine {
         // #1768: project a deterministic full-µs UTC string alongside updated_at.
         // to_char (not ::text — DateStyle-fragile) so extractStaleFromDB can stamp
         // links_extracted_at = the exact updated_at and the staleness predicate clears.
-        `SELECT id, slug, source_id, type, title, compiled_truth, timeline, frontmatter, updated_at,
+        `SELECT id, slug, source_id, type, title, compiled_truth, timeline, frontmatter,
+                effective_date, effective_date_source, updated_at,
                 to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS updated_at_iso
            FROM pages
            WHERE ${where}${afterClause}

src/core/types.ts

@@ -626,6 +626,10 @@ export interface StalePageRow {
   compiled_truth: string;
   timeline: string;
   frontmatter: Record<string, unknown>;
+  /** Page-level content date used as the fallback structured timeline anchor. */
+  effective_date: Date | null;
+  /** Provenance for effective_date (event_date/date/published/filename/fallback). */
+  effective_date_source: EffectiveDateSource | null;
   updated_at: Date;
   /**
    * Full-precision (microsecond) UTC ISO string of `updated_at`, projected

src/core/utils.ts

@@ -163,6 +163,8 @@ export function rowToStalePage(row: Record<string, unknown>): StalePageRow {
     compiled_truth: (row.compiled_truth as string | null) ?? '',
     timeline: (row.timeline as string | null) ?? '',
     frontmatter: (fm == null ? {} : (typeof fm === 'string' ? JSON.parse(fm) : fm)) as Record<string, unknown>,
+    effective_date: row.effective_date == null ? null : new Date(row.effective_date as string),
+    effective_date_source: (row.effective_date_source as StalePageRow['effective_date_source'] | null) ?? null,
     updated_at: new Date(row.updated_at as string),
     // #1768: full-µs UTC string projected by the SELECT (`updated_at_iso`).
     // Fallback derives an ISO string from the Date — NEVER String(Date), which

test/extract-db.test.ts

@@ -163,6 +163,35 @@ describe('gbrain extract timeline --source db', () => {
     expect(entries.map(e => e.summary).sort()).toEqual(['Closed Series A', 'Joined as CEO']);
   });
 
+  test('creates a fallback entry from page effective_date when content has no timeline row', async () => {
+    await engine.putPage('people/alice', {
+      type: 'person', title: 'Alice', compiled_truth: 'Alice is the CEO.', timeline: '',
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: 'date',
+    });
+
+    await runExtract(engine, ['timeline', '--source', 'db']);
+
+    const entries = await engine.getTimeline('people/alice');
+    expect(entries).toHaveLength(1);
+    expect(new Date(entries[0].date).toISOString().slice(0, 10)).toBe('2026-07-16');
+    expect(entries[0].summary).toBe('Alice');
+    expect(entries[0].source).toBe('page.effective_date:date');
+  });
+
+  test('does NOT create a fallback entry for mtime-derived (fallback) provenance', async () => {
+    await engine.putPage('people/bob', {
+      type: 'person', title: 'Bob', compiled_truth: 'Bob is the CTO.', timeline: '',
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: 'fallback',
+    });
+
+    await runExtract(engine, ['timeline', '--source', 'db']);
+
+    const entries = await engine.getTimeline('people/bob');
+    expect(entries).toHaveLength(0);
+  });
+
   test('idempotent via DB constraint', async () => {
     await engine.putPage('people/alice', {
       type: 'person', title: 'Alice', compiled_truth: '',

test/extract-stale.test.ts

@@ -90,6 +90,8 @@ describe('engine: stale-page extraction methods', () => {
     expect(batch1[0].compiled_truth).toBeTruthy();
     expect(batch1[0].title).toBeTruthy();
     expect(batch1[0].frontmatter).toBeDefined();
+    expect(batch1[0].effective_date).toBeNull();
+    expect(batch1[0].effective_date_source).toBeNull();
     const batch2 = await engine.listStalePagesForExtraction({ batchSize: 10, afterPageId: batch1[0].id });
     expect(batch2.length).toBe(1);
     expect(batch2[0].id).toBeGreaterThan(batch1[0].id);
@@ -102,6 +104,25 @@ describe('engine: stale-page extraction methods', () => {
 });
 
 describe('gbrain extract --stale', () => {
+  test('creates the effective_date fallback timeline row and remains idempotent', async () => {
+    await engine.putPage('people/alice', {
+      ...personPage('Alice'),
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: 'date',
+    });
+
+    await runExtract(engine, ['--stale']);
+    let entries = await engine.getTimeline('people/alice');
+    expect(entries).toHaveLength(1);
+    expect(new Date(entries[0].date).toISOString().slice(0, 10)).toBe('2026-07-16');
+    expect(entries[0].source).toBe('page.effective_date:date');
+
+    await engine.executeRaw(`UPDATE pages SET links_extracted_at = NULL WHERE slug = 'people/alice'`);
+    await runExtract(engine, ['--stale']);
+    entries = await engine.getTimeline('people/alice');
+    expect(entries).toHaveLength(1);
+  });
+
   test('extracts typed edges + stamps every processed page (incl. zero-link)', async () => {
     await engine.putPage('people/alice', personPage('Alice'));
     await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) is the CEO of [Acme](companies/acme).'));

test/extract.test.ts

@@ -3,6 +3,7 @@ import {
   extractMarkdownLinks,
   extractLinksFromFile,
   extractTimelineFromContent,
+  extractTimelineEntriesForPage,
   walkMarkdownFiles,
 } from '../src/commands/extract.ts';
 
@@ -187,6 +188,59 @@ describe('extractTimelineFromContent', () => {
   });
 });
 
+describe('extractTimelineEntriesForPage', () => {
+  it('falls back to effective_date when markdown has no explicit timeline row', () => {
+    const entries = extractTimelineEntriesForPage('No timeline here.', {
+      slug: 'people/test',
+      title: 'Test Person',
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: 'date',
+    });
+    expect(entries).toEqual([{
+      date: '2026-07-16',
+      source: 'page.effective_date:date',
+      summary: 'Test Person',
+      detail: '',
+    }]);
+  });
+
+  it('skips mtime-derived (fallback) provenance — no noise row per imported page', () => {
+    // computeEffectiveDate stamps source='fallback' (updated_at/created_at)
+    // on nearly every imported page; emitting a row for it would flood the
+    // timeline with one mtime-dated entry per timeline-less page brain-wide.
+    const entries = extractTimelineEntriesForPage('No timeline here.', {
+      slug: 'people/test',
+      title: 'Test Person',
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: 'fallback',
+    });
+    expect(entries).toEqual([]);
+  });
+
+  it('skips null/unknown provenance', () => {
+    const entries = extractTimelineEntriesForPage('No timeline here.', {
+      slug: 'people/test',
+      title: 'Test Person',
+      effective_date: new Date('2026-07-16T21:08:00Z'),
+      effective_date_source: null,
+    });
+    expect(entries).toEqual([]);
+  });
+
+  it('prefers explicit timeline rows over the effective_date fallback', () => {
+    const entries = extractTimelineEntriesForPage('- **2026-07-15** | Explicit event', {
+      slug: 'people/test',
+      title: 'Test Person',
+      effective_date: new Date('2026-07-16T00:00:00Z'),
+      effective_date_source: 'date',
+    });
+    expect(entries).toHaveLength(1);
+    expect(entries[0].date).toBe('2026-07-15');
+    expect(entries[0].summary).toBe('Explicit event');
+    expect(entries[0].source).toBeUndefined();
+  });
+});
+
 describe('walkMarkdownFiles', () => {
   it('is a function', () => {
     expect(typeof walkMarkdownFiles).toBe('function');