fix(conversation): extract iMessage facts with real timestamps (#2756) (#2958)

Co-authored-by: Sameer Bopardikar <203024074+sameerbopardikar@users.noreply.github.com>

src/commands/doctor.ts

@@ -3056,7 +3056,7 @@ export async function computeConversationFactsBacklogCheck(
     const typesRaw = await engine.getConfig(
       'cycle.conversation_facts_backfill.types',
     );
-    let types = ['conversation', 'meeting', 'slack', 'email'];
+    let types = ['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'];
     if (typesRaw) {
       try {
         const parsed = JSON.parse(typesRaw);
@@ -4927,8 +4927,8 @@ export async function buildChecks(
     try {
       const { readConversationBodyForParsing } = await import('../core/conversation-parser/body.ts');
       const { parseConversation } = await import('../core/conversation-parser/parse.ts');
-      const allowedTypes = ['conversation', 'meeting', 'slack', 'email'] as const;
-      // PageFilters supports singular `type` only; iterate the 4 types
+      const allowedTypes = ['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'] as const;
+      // PageFilters supports singular `type` only; iterate the allowed types
       // and cap at ~50/each to land at ~200 total max.
       const sample: import('../core/types.ts').Page[] = [];
       for (const t of allowedTypes) {

src/commands/extract-conversation-facts.ts

@@ -140,7 +140,14 @@ export const DEFAULT_MAX_COST_USD = 5.0;
  * `--types` flag is an explicit per-run override; cycle config is
  * the single source of truth.
  */
-export const ALLOWED_TYPES = ['conversation', 'meeting', 'slack', 'email'] as const;
+export const ALLOWED_TYPES = [
+  'conversation',
+  'meeting',
+  'slack',
+  'email',
+  'imessage',
+  'imessage-daily',
+] as const;
 export type AllowedType = (typeof ALLOWED_TYPES)[number];
 
 /**
@@ -756,6 +763,12 @@ async function processPage(
         source_markdown_slug: page.slug,
         source: PER_SEGMENT_SOURCE_PREFIX,
         source_session: sessionId,
+        // Preserve the conversation's valid time instead of defaulting every
+        // extracted fact to extraction time. Epoch-anchored parses have no
+        // trustworthy date, so they retain the existing now() fallback.
+        ...(seg.startIso && !seg.startIso.startsWith('1970-')
+          ? { valid_from: new Date(seg.startIso) }
+          : {}),
         context:
           fact.context ?? `from ${page.slug} segment ${seg.startIso}..${seg.endIso}`,
       }));

src/commands/jobs.ts

@@ -1497,7 +1497,7 @@ export async function registerBuiltinHandlers(
     }
     const types = Array.isArray(job.data.types)
       ? (job.data.types as string[]).filter((t) =>
-          ['conversation', 'meeting', 'slack', 'email'].includes(t),
+          ['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'].includes(t),
         )
       : undefined;
     const result = await runExtractConversationFactsCore(engine, {

src/commands/sources.ts

@@ -1183,7 +1183,14 @@ async function runAudit(engine: BrainEngine, args: string[]): Promise<void> {
   // frontmatter.type and estimates per-page segment count from body
   // bytes. Estimated per-segment Sonnet cost is a rough heuristic
   // (~2000 in + 500 out tokens at $3/MTok in + $15/MTok out ≈ $0.013).
-  const FACTS_BACKFILL_ALLOWED = ['conversation', 'meeting', 'slack', 'email'];
+  const FACTS_BACKFILL_ALLOWED = [
+    'conversation',
+    'meeting',
+    'slack',
+    'email',
+    'imessage',
+    'imessage-daily',
+  ];
   const FACTS_BACKFILL_CHARS_PER_SEGMENT = 6500; // matches SEGMENT_TEXT_CHAR_LIMIT
   const FACTS_BACKFILL_USD_PER_SEGMENT = 0.013;
   let factsBackfillPages = 0;

src/core/conversation-parser/builtins.ts

@@ -1,7 +1,7 @@
 /**
  * v0.41.16.0 — Built-in conversation parser pattern registry.
  *
- * Fourteen hand-vetted patterns covering the chat-export formats this
+ * Fifteen hand-vetted patterns covering the chat-export formats this
  * codebase is most likely to encounter. Each pattern's regex was
  * derived from a public format reference (source_doc field) so future
  * maintainers can verify against the wild shape.
@@ -50,7 +50,7 @@ export function cleanSpeaker(raw: string, override?: RegExp): string {
   return stripped || raw.trim();
 }
 
-/** The 14 hand-vetted built-in patterns. */
+/** The 15 hand-vetted built-in patterns. */
 export const BUILTIN_PATTERNS: readonly PatternEntry[] = [
   // -------------------------------------------------------------------
   // INLINE-DATE patterns (date in every line; less ambiguous; tried first).
@@ -178,6 +178,41 @@ export const BUILTIN_PATTERNS: readonly PatternEntry[] = [
       'OpenClaw meeting-ingestion pipeline reformat of Circleback transcripts (see your OpenClaw skills/meeting-ingestion/SKILL.md)',
   },
 
+  {
+    // iMessage sync's time-only 12-hour shape. AM/PM is required so this
+    // cannot shadow bold-paren-time's 24-hour form or imessage-slack's
+    // full-date form.
+    id: 'bold-paren-time-12h',
+    origin: 'builtin',
+    regex: /^\*\*(.+?)\*\*\s*\((\d{1,2}):(\d{2})\s*(AM|PM|am|pm)\)\s*:\s*(.*)$/,
+    captures: {
+      speaker_group: 1,
+      hour_group: 2,
+      minute_group: 3,
+      ampm_group: 4,
+      text_group: 5,
+    },
+    date_source: 'frontmatter',
+    time_format: '12h_ampm',
+    timezone_policy: 'utc_assumed_with_warn',
+    multi_line: false,
+    quick_reject: /^\*\*/,
+    test_positive: [
+      '**Me** (9:04 AM): sounds good, see you then',
+      '**+155****0135** (9:39 AM): Will do',
+      '**Alice Example** (12:00 PM): noon message',
+      '**Bob Example** (5:38 pm): lowercase ampm',
+    ],
+    test_negative: [
+      '**Alice** (00:00): 24h shape',
+      '**Alice Example** (2024-03-15 9:00 AM): full-date iMessage shape',
+      '**[18:37] G T:** telegram bracket',
+      'Alice (9:00 AM): missing the bold',
+    ],
+    source_doc:
+      'Time-only 12h AM/PM iMessage export shape: `**Speaker** (H:MM AM): text`',
+  },
+
   {
     // Fathom/phone-call raw transcripts in this workspace use a plain
     // `Speaker A: ...` / `Speaker B: ...` shape with no per-line time.

src/core/conversation-parser/parse.ts

@@ -321,11 +321,22 @@ export function applyPattern(
   if (!body) return [];
   const out: MatchedMessage[] = [];
   const lines = body.split(/\r?\n/);
+  // Some multi-day conversation exports use markdown date headings instead
+  // of repeating a date on every message. Keep the caller's context immutable
+  // while advancing a local date anchor as those headings are encountered.
+  const runningCtx: DateContext = { ...dateCtx };
+  const dateHeaderRe = /^#{1,4}\s+(\d{4}-\d{2}-\d{2})\s*$/;
   for (let i = 0; i < lines.length; i++) {
     const rawLine = lines[i];
     const line = rawLine.trim();
     if (!line) continue;
 
+    const dateHeader = dateHeaderRe.exec(line);
+    if (dateHeader) {
+      runningCtx.fallbackDate = dateHeader[1];
+      continue;
+    }
+
     // Quick-reject fast path.
     if (entry.quick_reject && !entry.quick_reject.test(line)) {
       // Continuation handling for orphan lines.
@@ -339,7 +350,7 @@ export function applyPattern(
 
     const m = entry.regex.exec(line);
     if (m) {
-      const iso = buildIso(m, entry, dateCtx);
+      const iso = buildIso(m, entry, runningCtx);
       if (iso === null) continue; // reconstruction failed; skip line
       const rawSpeaker = m[entry.captures.speaker_group] ?? '';
       const speaker = cleanSpeaker(rawSpeaker, entry.speaker_clean);
@@ -380,7 +391,7 @@ function getNonBlankLines(body: string, headCap?: number): string[] {
  * window) and `scorePatternFull` (whole body) delegate here so the
  * quick_reject + regex loop lives in one place. Reused by
  * `parseConversation`'s fallback path which pre-splits ONCE and
- * passes the array to all 12 candidates (saves 11 redundant body
+ * passes the array to all 15 candidates (saves 14 redundant body
  * splits per fallback pass).
  */
 function scoreFromLines(

test/conversation-parser-cli.test.ts

@@ -77,7 +77,7 @@ describe('runConversationParser — help', () => {
 });
 
 describe('runConversationParser — list-builtins', () => {
-  test('human output includes all 12 pattern ids', async () => {
+  test('human output includes all built-in pattern ids', async () => {
     const cap = captureStdio();
     try {
       await runConversationParser(null, ['list-builtins']);

test/conversation-parser/parse.test.ts

@@ -3,7 +3,7 @@
  *
  * Covers:
  *   - PR #1461's 6 telegram-bracket cases verbatim (REGRESSION pin)
- *   - All 12 built-in patterns hit their test_positive samples
+ *   - All built-in patterns hit their test_positive samples
  *   - Date derivation precedence (D8)
  *   - Pattern priority scoring (D18) — overlap resolution
  *   - Quick-reject fast path (D11)
@@ -116,7 +116,7 @@ describe('parseConversation — REGRESSION PR #1461 (telegram-bracket)', () => {
 });
 
 // ---------------------------------------------------------------------------
-// All 12 built-ins must parse their test_positive samples
+// All built-ins must parse their test_positive samples
 // ---------------------------------------------------------------------------
 
 describe('parseConversation — every built-in matches its test_positive sample', () => {
@@ -261,6 +261,40 @@ describe('parseConversation — multi-line continuation (D5)', () => {
   });
 });
 
+describe('parseConversation — iMessage time-only 12h and date headings (#2756)', () => {
+  test('parses the time-only 12-hour iMessage shape', () => {
+    const r = parseConversation('**Alice Example** (9:04 PM): hello', {
+      fallbackDate: '2024-03-15',
+    });
+    expect(r.matched_pattern_id).toBe('bold-paren-time-12h');
+    expect(r.messages).toHaveLength(1);
+    expect(r.messages[0].timestamp).toBe('2024-03-15T21:04:00Z');
+  });
+
+  test('markdown date headings advance the running date without becoming message text', () => {
+    const body = [
+      '## 2024-03-15',
+      '**Alice Example** (9:04 AM): first day',
+      '## 2024-03-16',
+      '**Bob Example** (10:05 PM): second day',
+    ].join('\n');
+    const r = parseConversation(body, { fallbackDate: '2024-03-01' });
+    expect(r.matched_pattern_id).toBe('bold-paren-time-12h');
+    expect(r.messages.map((m) => m.timestamp)).toEqual([
+      '2024-03-15T09:04:00Z',
+      '2024-03-16T22:05:00Z',
+    ]);
+    expect(r.messages[0].text).toBe('first day');
+  });
+
+  test('date headings do not mutate the caller-provided context', () => {
+    const ctx = { fallbackDate: '2024-03-01', source: 'explicit' as const };
+    const pattern = BUILTIN_PATTERNS.find((p) => p.id === 'bold-paren-time-12h')!;
+    applyPattern('## 2024-03-16\n**Alice** (9:04 AM): hello', pattern, ctx);
+    expect(ctx.fallbackDate).toBe('2024-03-01');
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Timezone warning (D19)
 // ---------------------------------------------------------------------------

test/e2e/conversation-parser-pglite.test.ts

@@ -2,7 +2,7 @@
  * v0.41.16.0 — E2E test for the conversation parser cathedral against
  * a real PGLite brain.
  *
- * For each of the 12 built-in formats: seed a page through
+ * For each built-in format: seed a page through
  * `importFromContent`, run `parseConversation` against the body, assert
  * the parser identifies the correct pattern AND produces at least one
  * message AND the message timestamp lands in the expected date range.

test/extract-conversation-facts.test.ts

@@ -36,6 +36,7 @@ import {
   MAX_PAGE_BODY_BYTES,
   TERMINAL_AUDIT_SOURCE,
   PER_SEGMENT_SOURCE_PREFIX,
+  ALLOWED_TYPES,
 } from '../src/commands/extract-conversation-facts.ts';
 
 // ---------------------------------------------------------------------------
@@ -93,6 +94,11 @@ describe('parseConversationMessages', () => {
   });
 });
 
+test('conversation-facts allowlist includes native iMessage page types (#2756)', () => {
+  expect(ALLOWED_TYPES).toContain('imessage');
+  expect(ALLOWED_TYPES).toContain('imessage-daily');
+});
+
 // ---------------------------------------------------------------------------
 // splitIntoSegments — PR's 5 cases verbatim plus tuning regression.
 // ---------------------------------------------------------------------------
@@ -319,6 +325,13 @@ describe('runExtractConversationFactsCore', () => {
       timeline: '',
       frontmatter: {},
     });
+    await engine.putPage('conversations/imessage/native-example', {
+      type: 'imessage',
+      title: 'Native iMessage export',
+      compiled_truth: SAMPLE_BODY,
+      timeline: '',
+      frontmatter: {},
+    });
     await engine.putPage('people/alice-example', {
       type: 'person',
       title: 'Alice Example',
@@ -392,6 +405,17 @@ describe('runExtractConversationFactsCore', () => {
     expect(result.pages_considered).toBe(0);
   });
 
+  test('native imessage page types are eligible by default', async () => {
+    const result = await runExtractConversationFactsCore(engine, {
+      sourceId: 'default',
+      slug: 'conversations/imessage/native-example',
+      dryRun: true,
+      sleepMs: 0,
+    });
+    expect(result.pages_considered).toBe(1);
+    expect(result.pages_processed).toBe(1);
+  });
+
   test('sinceIso filters already-processed history', async () => {
     const result = await runExtractConversationFactsCore(engine, {
       sourceId: 'default',
@@ -432,6 +456,17 @@ describe('runExtractConversationFactsCore', () => {
     );
     expect(Number(perSegFacts[0]?.count ?? 0)).toBeGreaterThan(0);
 
+    const validTimes = await engine.executeRaw<{ valid_from: Date }>(
+      `SELECT valid_from FROM facts
+       WHERE source = $1 AND source_session = $2
+       ORDER BY valid_from ASC`,
+      [PER_SEGMENT_SOURCE_PREFIX, `${PER_SEGMENT_SOURCE_PREFIX}:conversations/imessage/alice-example`],
+    );
+    expect(validTimes.map((row) => new Date(row.valid_from).toISOString())).toEqual([
+      '2024-03-15T09:00:00.000Z',
+      '2024-03-16T08:00:00.000Z',
+    ]);
+
     // Terminal audit row present.
     const terminalRows = await engine.executeRaw<{ count: string | number }>(
       `SELECT COUNT(*) AS count FROM facts WHERE source = $1 AND source_session = $2`,

test/fixtures/conversation-formats/all.jsonl

@@ -1,5 +1,6 @@
 {"fixture_id":"imessage-001","pattern":"imessage-slack","frontmatter":{"date":"2024-03-15"},"body":"**Alice Example** (2024-03-15 9:00 AM): morning\n**Bob Example** (2024-03-15 9:01 AM): hey there\n**Alice Example** (2024-03-15 9:02 AM): how are you\n**Bob Example** (2024-03-15 9:03 AM): good thanks\n**Alice Example** (2024-03-15 9:04 AM): you?","expected_messages":5,"expected_participants":["Alice Example","Bob Example"]}
 {"fixture_id":"imessage-002","pattern":"imessage-slack","frontmatter":{"date":"2024-03-15"},"body":"**Charlie Example** (2024-03-15 2:00 PM): afternoon\n**Charlie Example** (2024-03-15 2:01 PM): are you there?\n**Diana Example** (2024-03-15 2:05 PM): yes\n**Charlie Example** (2024-03-15 2:06 PM): great","expected_messages":4,"expected_participants":["Charlie Example","Diana Example"]}
+{"fixture_id":"imessage-time-only-12h-001","pattern":"bold-paren-time-12h","frontmatter":{"date":"2024-03-15"},"body":"## 2024-03-15\n**Alice Example** (9:04 AM): morning\n**Bob Example** (9:05 AM): hey there\n## 2024-03-16\n**Alice Example** (10:06 PM): second day\n**Bob Example** (10:07 PM): good night","expected_messages":4,"expected_participants":["Alice Example","Bob Example"]}
 {"fixture_id":"telegram-bracket-001","pattern":"telegram-bracket","frontmatter":{"date":"2026-05-24","timezone":"America/Los_Angeles"},"body":"**[18:37] 👤 Alice Example:** hello world\n**[18:38] 👤 Bob Example:** hey\n**[18:39] 👤 Alice Example:** how are you\n**[18:40] 👤 Bob Example:** good","expected_messages":4,"expected_participants":["Alice Example","Bob Example"]}
 {"fixture_id":"telegram-bracket-002","pattern":"telegram-bracket","frontmatter":{"date":"2026-05-25","timezone":"America/Los_Angeles"},"body":"**[06:00] 🤖 Zion Bot:** On it.\n**[06:01] 👤 Charlie Example:** thanks\n**[06:02] 🤖 Zion Bot:** anything else?\n**[06:03] 👤 Charlie Example:** no good","expected_messages":4,"expected_participants":["Zion Bot","Charlie Example"]}
 {"fixture_id":"whatsapp-iso-001","pattern":"whatsapp-iso","frontmatter":{"date":"2024-03-15"},"body":"[15/03/24, 18:37:00] Alice Example: hello\n[15/03/24, 18:37:30] Bob Example: hey\n[15/03/24, 18:38:00] Alice Example: how are you\n[15/03/24, 18:39:00] Bob Example: good","expected_messages":4,"expected_participants":["Alice Example","Bob Example"]}

test/fixtures/conversation-formats/imessage-time-only-12h.jsonl

@@ -0,0 +1 @@
+{"fixture_id":"imessage-time-only-12h-001","pattern":"bold-paren-time-12h","frontmatter":{"date":"2024-03-15"},"body":"## 2024-03-15\n**Alice Example** (9:04 AM): morning\n**Bob Example** (9:05 AM): hey there\n## 2024-03-16\n**Alice Example** (10:06 PM): second day\n**Bob Example** (10:07 PM): good night","expected_messages":4,"expected_participants":["Alice Example","Bob Example"]}