fix(migrate): preserve sources and scope resume targets (#2677) (#2736)

migrate --to now copies the complete source catalog before pages (fixes the pages_source_id_fkey failure on multi-source brains), and resume manifests carry an opaque target identity so a checkpoint from one target is discarded for a different target. Fixes #2677.

src/commands/migrate-engine.ts

@@ -12,6 +12,8 @@ import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabas
 import type { BrainEngine } from '../core/engine.ts';
 import type { EngineConfig } from '../core/types.ts';
 import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
+import { createHash } from 'crypto';
+import { resolve } from 'path';
 import { createProgress } from '../core/progress.ts';
 import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
 
@@ -49,12 +51,27 @@ function getManifestPath(): string {
   return gbrainPath('migrate-manifest.json');
 }
 
-interface MigrateManifest {
+export interface MigrateManifest {
   completed_slugs: string[];
   target_engine: string;
+  target_id?: string;
+  schema_version?: number;
   started_at: string;
 }
 
+export function migrationTargetId(config: EngineConfig): string {
+  const locator = config.engine === 'postgres'
+    ? config.database_url ?? ''
+    : resolve(config.database_path ?? gbrainPath('brain.pglite'));
+  return createHash('sha256')
+    .update(JSON.stringify([config.engine, locator]))
+    .digest('hex');
+}
+
+export function manifestMatchesTarget(manifest: MigrateManifest, targetId: string): boolean {
+  return manifest.schema_version === 2 && manifest.target_id === targetId;
+}
+
 function loadManifest(): MigrateManifest | null {
   const path = getManifestPath();
   if (!existsSync(path)) return null;
@@ -74,6 +91,58 @@ function clearManifest(): void {
   if (existsSync(path)) unlinkSync(path);
 }
 
+interface MigratedSourceRow {
+  id: string;
+  name: string;
+  local_path: string | null;
+  last_commit: string | null;
+  last_sync_at: Date | string | null;
+  config_json: string;
+  archived: boolean;
+  archived_at: Date | string | null;
+  archive_expires_at: Date | string | null;
+  contextual_retrieval_mode: string | null;
+  trust_frontmatter_overrides: boolean;
+  newest_content_at: Date | string | null;
+  created_at: Date | string;
+}
+
+export async function copyMigrationSources(source: BrainEngine, target: BrainEngine): Promise<void> {
+  const sources = await source.executeRaw<MigratedSourceRow>(`
+    SELECT id, name, local_path, last_commit, last_sync_at, config::text AS config_json, archived,
+           archived_at, archive_expires_at, contextual_retrieval_mode,
+           trust_frontmatter_overrides, newest_content_at, created_at
+      FROM sources
+     ORDER BY (id = 'default') DESC, id`);
+
+  for (const row of sources) {
+    await target.executeRaw(`
+      INSERT INTO sources
+        (id, name, local_path, last_commit, last_sync_at, config, archived,
+         archived_at, archive_expires_at, contextual_retrieval_mode,
+         trust_frontmatter_overrides, newest_content_at, created_at)
+      VALUES ($1, $2, $3, $4, $5, $6::text::jsonb, $7, $8, $9, $10, $11, $12, $13)
+      ON CONFLICT (id) DO UPDATE SET
+        name = EXCLUDED.name,
+        local_path = EXCLUDED.local_path,
+        last_commit = EXCLUDED.last_commit,
+        last_sync_at = EXCLUDED.last_sync_at,
+        config = EXCLUDED.config,
+        archived = EXCLUDED.archived,
+        archived_at = EXCLUDED.archived_at,
+        archive_expires_at = EXCLUDED.archive_expires_at,
+        contextual_retrieval_mode = EXCLUDED.contextual_retrieval_mode,
+        trust_frontmatter_overrides = EXCLUDED.trust_frontmatter_overrides,
+        newest_content_at = EXCLUDED.newest_content_at,
+        created_at = EXCLUDED.created_at`, [
+      row.id, row.name, row.local_path, row.last_commit, row.last_sync_at,
+      row.config_json, row.archived, row.archived_at, row.archive_expires_at,
+      row.contextual_retrieval_mode, row.trust_frontmatter_overrides,
+      row.newest_content_at, row.created_at,
+    ]);
+  }
+}
+
 export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]): Promise<void> {
   const opts = parseArgs(args);
   const config = loadConfig();
@@ -100,6 +169,7 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
   } else {
     targetConfig.database_path = opts.targetPath || gbrainPath('brain.pglite');
   }
+  const targetId = migrationTargetId(targetConfig);
 
   // Connect to target
   console.log(`Connecting to target (${opts.targetEngine})...`);
@@ -129,7 +199,7 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
 
   // Load or create manifest for resume
   let manifest = loadManifest();
-  if (manifest && manifest.target_engine !== opts.targetEngine) {
+  if (manifest && !manifestMatchesTarget(manifest, targetId)) {
     console.log('Previous migration was to a different target. Starting fresh.');
     manifest = null;
   }
@@ -144,10 +214,17 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
     manifest = {
       completed_slugs: [],
       target_engine: opts.targetEngine,
+      target_id: targetId,
+      schema_version: 2,
       started_at: new Date().toISOString(),
     };
   }
 
+  // Pages.source_id is a foreign key. Copy the complete source catalog first,
+  // including archived rows and sync/routing metadata, so every page write has
+  // a valid parent and the target preserves source behavior.
+  await copyMigrationSources(sourceEngine, targetEngine);
+
   // Get all source pages
   const sourceStats = await sourceEngine.getStats();
   const allPages = await sourceEngine.listPages({ limit: 100000 });

test/e2e/migrate-engine-sources-postgres.test.ts

@@ -0,0 +1,44 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
+import { copyMigrationSources } from '../../src/commands/migrate-engine.ts';
+import { hasDatabase, setupDB, teardownDB, getEngine } from './helpers.ts';
+
+const describePg = hasDatabase() ? describe : describe.skip;
+
+describePg('migrate-engine source copy PGLite to Postgres', () => {
+  let source: PGLiteEngine;
+
+  beforeAll(async () => {
+    await setupDB();
+    source = new PGLiteEngine();
+    await source.connect({});
+    await source.initSchema();
+  });
+
+  afterAll(async () => {
+    if (source) await source.disconnect();
+    await teardownDB();
+  });
+
+  test('copies source parents before overlapping-slug pages', async () => {
+    await source.executeRaw(`INSERT INTO sources (id, name, config)
+      VALUES ('source-a', 'Source A', '{"federated":true}'::jsonb),
+             ('source-b', 'Source B', '{"federated":false}'::jsonb)`);
+    for (const sourceId of ['source-a', 'source-b']) {
+      await source.putPage('people/shared', {
+        type: 'person', title: sourceId, compiled_truth: sourceId,
+      }, { sourceId });
+    }
+
+    const target = getEngine();
+    await copyMigrationSources(source, target);
+    for (const page of await source.listPages({ limit: 10 })) {
+      await target.putPage(page.slug, {
+        type: page.type, title: page.title, compiled_truth: page.compiled_truth,
+      }, { sourceId: page.source_id });
+    }
+
+    expect(await target.getPage('people/shared', { sourceId: 'source-a' })).not.toBeNull();
+    expect(await target.getPage('people/shared', { sourceId: 'source-b' })).not.toBeNull();
+  });
+});

test/e2e/multi-source-bug-class.test.ts

@@ -33,21 +33,28 @@ import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
 import { resetPgliteState } from '../helpers/reset-pglite.ts';
 import { validateSourceId } from '../../src/core/utils.ts';
 import { extractTakesFromDb } from '../../src/core/cycle/extract-takes.ts';
+import { copyMigrationSources } from '../../src/commands/migrate-engine.ts';
 
 let engine: PGLiteEngine;
+let migrationTarget: PGLiteEngine;
 
 beforeAll(async () => {
   engine = new PGLiteEngine();
   await engine.connect({} as never);
   await engine.initSchema();
+  migrationTarget = new PGLiteEngine();
+  await migrationTarget.connect({} as never);
+  await migrationTarget.initSchema();
 });
 
 afterAll(async () => {
   if (engine) await engine.disconnect();
+  if (migrationTarget) await migrationTarget.disconnect();
 });
 
 beforeEach(async () => {
   await resetPgliteState(engine);
+  await resetPgliteState(migrationTarget);
   // Seed second source row. Default source is seeded by resetPgliteState.
   await engine.executeRaw(
     `INSERT INTO sources (id, name, config)
@@ -75,6 +82,48 @@ beforeEach(async () => {
 });
 
 describe('multi-source bug class', () => {
+  test('migration copies source metadata before overlapping-slug pages', async () => {
+    await engine.executeRaw(`
+      UPDATE sources SET
+        local_path = '/tmp/media-corpus', last_commit = 'commit-media',
+        last_sync_at = '2026-07-02T00:00:00Z',
+        config = '{"federated":false,"custom":"preserved"}'::jsonb,
+        archived = true, archived_at = '2026-07-03T00:00:00Z',
+        archive_expires_at = '2026-08-03T00:00:00Z',
+        contextual_retrieval_mode = 'tokenmax',
+        trust_frontmatter_overrides = true,
+        newest_content_at = '2026-07-01T00:00:00Z'
+      WHERE id = 'media-corpus'`);
+
+    await copyMigrationSources(engine, migrationTarget);
+    const sourceRows = await migrationTarget.executeRaw<any>(
+      `SELECT * FROM sources WHERE id = 'media-corpus'`,
+    );
+    expect(sourceRows).toHaveLength(1);
+    expect(sourceRows[0].last_commit).toBe('commit-media');
+    expect(sourceRows[0].archived).toBe(true);
+    expect(sourceRows[0].contextual_retrieval_mode).toBe('tokenmax');
+    expect(sourceRows[0].trust_frontmatter_overrides).toBe(true);
+    const config = typeof sourceRows[0].config === 'string'
+      ? JSON.parse(sourceRows[0].config)
+      : sourceRows[0].config;
+    expect(config.custom).toBe('preserved');
+
+    const overlapping = await engine.listPages({ limit: 100 });
+    for (const page of overlapping) {
+      await migrationTarget.putPage(page.slug, {
+        type: page.type,
+        title: page.title,
+        compiled_truth: page.compiled_truth,
+        timeline: page.timeline,
+        frontmatter: page.frontmatter,
+        content_hash: page.content_hash,
+      }, { sourceId: page.source_id });
+    }
+    expect(await migrationTarget.getPage('people/alice', { sourceId: 'default' })).not.toBeNull();
+    expect(await migrationTarget.getPage('people/alice', { sourceId: 'media-corpus' })).not.toBeNull();
+  });
+
   test('listAllPageRefs returns one row per (slug, source_id), ordered (F11)', async () => {
     const refs = await engine.listAllPageRefs();
     // 4 rows: alice@default, alice@media-corpus, widget@default, post-123@media-corpus

test/migrate-engine-resume.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  manifestMatchesTarget,
+  migrationTargetId,
+  type MigrateManifest,
+} from '../src/commands/migrate-engine.ts';
+
+describe('migrate-engine resume identity', () => {
+  test('crash manifest resumes only against the same PGLite target', () => {
+    const targetA = migrationTargetId({ engine: 'pglite', database_path: '/tmp/target-a' });
+    const targetB = migrationTargetId({ engine: 'pglite', database_path: '/tmp/target-b' });
+    const crashed: MigrateManifest = {
+      schema_version: 2,
+      target_engine: 'pglite',
+      target_id: targetA,
+      completed_slugs: ['source-a::people/shared'],
+      started_at: '2026-07-10T00:00:00.000Z',
+    };
+
+    expect(manifestMatchesTarget(crashed, targetA)).toBe(true);
+    expect(manifestMatchesTarget(crashed, targetB)).toBe(false);
+  });
+
+  test('legacy engine-only manifest cannot skip pages on a second target', () => {
+    const legacy: MigrateManifest = {
+      target_engine: 'postgres',
+      completed_slugs: ['people/shared'],
+      started_at: '2026-07-10T00:00:00.000Z',
+    };
+    const target = migrationTargetId({
+      engine: 'postgres',
+      database_url: 'postgresql://user:secret@db.example.invalid/brain-b',
+    });
+
+    expect(manifestMatchesTarget(legacy, target)).toBe(false);
+    expect(target).not.toContain('user');
+    expect(target).not.toContain('secret');
+    expect(target).not.toContain('db.example.invalid');
+  });
+});