fix(doctor): surface abandoned PGLite stores after an engine migration (#3856) (#3879)

Wave-assembled from PR #3879 by @Masashi-Ono0611.

Co-Authored-By: masashiono0611 <masashi.ono.0611@gmail.com>

src/commands/doctor.ts

@@ -6181,6 +6181,44 @@ export async function buildChecks(
     // Best-effort environment check; never block doctor.
   }
 
+  // 3g. pglite_leftovers (#3856). A pglite -> postgres migration leaves the
+  // old engine store (`brain.pglite/`) under the gbrain home forever — dead
+  // weight roughly the size of the live DB that nothing surfaces, silently
+  // riding along in any backup that archives the home dir. Assessment is a
+  // pure helper (src/core/pglite-leftovers-check.ts); it warns ONLY for a
+  // durable postgres engine, and skips while `migrate-manifest.json` exists
+  // (an in-flight/interrupted migration can make `brain.pglite` the LIVE
+  // target while the durable engine still reads postgres — #3194) and for
+  // everything else (fail open).
+  // The engine is read from config.json DIRECTLY, not loadConfig(): a
+  // transient DATABASE_URL (#427) can make a live PGLite brain resolve as
+  // postgres for one process, and deletion advice must never rest on an
+  // env override (Codex review P1).
+  // Warn-only by design: WHEN the abandoned store is safe to drop is a
+  // policy question (#3856), so the remediation is a verified manual delete
+  // — no CLI command is named that does not exist (#3697).
+  try {
+    const { readFileSync } = await import('node:fs');
+    const durableEngine = (
+      JSON.parse(readFileSync(join(gbrainPath(), 'config.json'), 'utf8')) as { engine?: unknown }
+    ).engine;
+    const { assessPgliteLeftovers } = await import('../core/pglite-leftovers-check.ts');
+    const leftovers = assessPgliteLeftovers(
+      typeof durableEngine === 'string' ? durableEngine : undefined,
+      gbrainPath(),
+    );
+    if (leftovers.status !== 'skip') {
+      checks.push({
+        name: 'pglite_leftovers',
+        status: leftovers.status,
+        message: leftovers.message,
+      });
+    }
+  } catch {
+    // Best-effort filesystem-hygiene check; never block doctor (a missing/
+    // unparseable config.json lands here and skips, same fail-open posture).
+  }
+
   // 3b-multi-source. Multi-source drift (v0.31.8 — D8 + D17 + OV12 + OV13).
   // Pre-v0.30.3 putPage misrouted multi-source writes to (default, slug).
   // For each non-default source with local_path set, walk the FS and surface

src/core/doctor-categories.ts

@@ -166,6 +166,7 @@ export const OPS_CHECK_NAMES: ReadonlySet<string> = new Set([
   'orphan_clones',
   'pgbouncer_prepare',
   'pglite_data_dir',
+  'pglite_leftovers',
   'pgvector',
   'pool_budget',
   'progressive_batch_audit_health',

src/core/pglite-leftovers-check.ts

@@ -0,0 +1,242 @@
+/**
+ * pglite-leftovers-check — detect abandoned PGLite stores after an engine
+ * migration (#3856).
+ *
+ * A pglite -> postgres migration leaves `brain.pglite/` (the old engine's
+ * store) under the gbrain home, untouched, forever. It weighs roughly as much
+ * as the live DB, nothing ever surfaces it, and it silently rides along in any
+ * backup that archives the gbrain home — the reporting brain paid to
+ * permanently archive a dead engine's corpse on Arweave for 2.5 months
+ * (~half the monthly cost, once excluded by hand).
+ *
+ * Pure assessment helpers (filesystem-only, no network, no shelling out) so
+ * `gbrain doctor` can warn with receipts, in the same shape as
+ * npm-squat-check.ts (#505). The caller supplies the engine kind — which MUST
+ * come from the DURABLE file config, not the resolved/env-overridden one: a
+ * transient `DATABASE_URL` (#427) can make a live PGLite brain look like
+ * postgres for one process, and deletion advice must never rest on that.
+ *
+ * Scope is deliberately `brain.pglite` alone — the one directory the engines
+ * themselves create. gbrain does NOT create pre-migrate safety copies or any
+ * other `brain.pglite.*` sibling (hand-made `.bak` copies, operator scripts,
+ * etc.); every such sibling has unknown provenance and is NOT claimed.
+ *
+ * Migration-in-flight gate: `gbrain migrate` banks resume progress in
+ * `migrate-manifest.json` at the gbrain home root and clears it only on clean
+ * completion — the same completion that flips the durable engine (#3194). So
+ * while the manifest exists, the durable engine can still read `postgres`
+ * even though `brain.pglite/` is the LIVE TARGET of an interrupted
+ * postgres -> pglite migration. Deleting it then would leave the manifest
+ * behind, and a re-run would silently skip the "already completed" pages
+ * against an empty store. The check therefore skips whenever the manifest is
+ * present and never assesses leftovers, let alone offers deletion advice.
+ *
+ * Deliberately warn-only with a MANUAL remediation: deciding when the old
+ * store is safe to drop is a policy question (#3856 ask 2) — this check only
+ * makes the corpse visible, it never deletes anything. The remediation text
+ * names no CLI command that does not exist (#3697).
+ */
+import { existsSync, lstatSync, opendirSync } from 'node:fs';
+import { join } from 'node:path';
+
+export interface PgliteLeftoverDir {
+  path: string;
+  /** Bytes summed over a bounded, symlink-free walk — a floor when size_incomplete. */
+  approx_bytes: number;
+  /** True when the walk hit its budget OR any entry was unreadable — approx_bytes is then a floor. */
+  size_incomplete: boolean;
+  /** The directory inode's own mtime (ISO) — labeled as such in the message: file
+   *  contents can change without touching the parent dir's mtime. */
+  dir_mtime: string | null;
+}
+
+export interface PgliteLeftoversAssessment {
+  status: 'ok' | 'warn' | 'skip';
+  message: string;
+  leftovers: PgliteLeftoverDir[];
+}
+
+/**
+ * Bound on how many directory entries the size walk visits across the WHOLE
+ * assessment (all leftover dirs share one budget). A PGLite store is a modest
+ * number of large files, so real stores finish well under this; the cap exists
+ * so a pathological tree cannot stall doctor's synchronous path.
+ */
+export const SIZE_WALK_MAX_ENTRIES = 20_000;
+
+/** The resume manifest `gbrain migrate` writes at the gbrain home root; it
+ *  survives an interrupted run and is cleared only on clean completion. */
+export const MIGRATE_MANIFEST_NAME = 'migrate-manifest.json';
+
+/** True only for the one directory the engines themselves create: the old
+ *  store `brain.pglite`. gbrain never creates `brain.pglite.*` siblings
+ *  (no pre-migrate copies, no backups) — those all have unknown provenance
+ *  and are out of scope. */
+export function isMigrationLeftoverName(name: string): boolean {
+  return name === 'brain.pglite';
+}
+
+/** Iterative, symlink-free size walk. Shares `budget` across calls; flags
+ *  incompleteness on budget exhaustion AND on any unreadable entry (an
+ *  unreadable multi-GB store must never be reported as exactly 0 B). */
+function walkSize(root: string, budget: { entries: number }): { bytes: number; incomplete: boolean } {
+  let bytes = 0;
+  let incomplete = false;
+  const stack: string[] = [root];
+  while (stack.length > 0) {
+    const dir = stack.pop() as string;
+    let handle: ReturnType<typeof opendirSync>;
+    try {
+      handle = opendirSync(dir);
+    } catch {
+      incomplete = true;
+      continue;
+    }
+    try {
+      for (;;) {
+        if (budget.entries <= 0) {
+          incomplete = true;
+          break;
+        }
+        let entry: ReturnType<typeof handle.readSync>;
+        try {
+          entry = handle.readSync();
+        } catch {
+          // opendirSync can succeed on a dir whose entries we cannot read
+          // (mode 000) — the EACCES surfaces here, not at open (measured).
+          incomplete = true;
+          break;
+        }
+        if (entry === null) break;
+        budget.entries--;
+        const p = join(dir, entry.name);
+        if (entry.isSymbolicLink()) continue; // never follow — the walk must not escape the store
+        if (entry.isDirectory()) {
+          stack.push(p);
+        } else if (entry.isFile()) {
+          try {
+            bytes += lstatSync(p).size;
+          } catch {
+            incomplete = true;
+          }
+        }
+      }
+    } finally {
+      try {
+        handle.closeSync();
+      } catch {
+        // already closed / racing unlink — nothing to do
+      }
+    }
+    if (budget.entries <= 0 && stack.length > 0) {
+      incomplete = true;
+      break;
+    }
+  }
+  return { bytes, incomplete };
+}
+
+/** Human-readable size. For floors (incomplete walks) the value is rounded
+ *  DOWN, so ">= 1.6 MB" can never display as ">= 2 MB". */
+function humanBytes(n: number, floor: boolean): string {
+  const fmt = (v: number, digits: number) => (floor ? Math.floor(v * 10 ** digits) / 10 ** digits : v).toFixed(digits);
+  if (n >= 1024 ** 3) return `${fmt(n / 1024 ** 3, 1)} GB`;
+  if (n >= 1024 ** 2) return `${fmt(n / 1024 ** 2, 0)} MB`;
+  if (n >= 1024) return `${fmt(n / 1024, 0)} KB`;
+  return `${n} B`;
+}
+
+/**
+ * Assess a gbrain home for abandoned PGLite migration leftovers.
+ *
+ * - engine `postgres` is the ONLY value that can warn: that is the migration
+ *   target #3856 documents, and the one case where `brain.pglite` is known
+ *   dead weight. `pglite` means the store is live; anything else (missing /
+ *   unreadable / future engines) skips — deletion advice must fail open.
+ * - `migrate-manifest.json` present at the home root -> `skip`, even on a
+ *   durable postgres engine: an engine migration is in progress or was
+ *   interrupted, and `brain.pglite` may be its live target (see file header).
+ * - postgres + no leftover dirs -> `ok`.
+ * - postgres + leftovers -> `warn`, naming each dir with a floor-marked
+ *   approximate size and its directory mtime.
+ *
+ * `engineKind` must be the DURABLE file-config engine (see file header).
+ * `maxEntries` exists for tests; production callers use the default.
+ */
+export function assessPgliteLeftovers(
+  engineKind: string | null | undefined,
+  gbrainHome: string,
+  maxEntries: number = SIZE_WALK_MAX_ENTRIES,
+): PgliteLeftoversAssessment {
+  if (engineKind !== 'postgres') {
+    return { status: 'skip', message: '', leftovers: [] };
+  }
+  if (existsSync(join(gbrainHome, MIGRATE_MANIFEST_NAME))) {
+    return {
+      status: 'skip',
+      message:
+        `${MIGRATE_MANIFEST_NAME} is present: an engine migration is in progress or was ` +
+        `interrupted, and brain.pglite may be the live migration target — not assessing leftovers.`,
+      leftovers: [],
+    };
+  }
+  let names: string[] = [];
+  const budget = { entries: maxEntries };
+  try {
+    const handle = opendirSync(gbrainHome);
+    try {
+      for (;;) {
+        const entry = handle.readSync();
+        if (entry === null) break;
+        // Top-level symlinks are skipped too: a symlinked brain.pglite is not
+        // a store this check can claim ownership of.
+        if (entry.isDirectory() && !entry.isSymbolicLink() && isMigrationLeftoverName(entry.name)) {
+          names.push(entry.name);
+        }
+      }
+    } finally {
+      handle.closeSync();
+    }
+  } catch {
+    return { status: 'skip', message: '', leftovers: [] };
+  }
+  const leftovers: PgliteLeftoverDir[] = [];
+  for (const name of names.sort()) {
+    const p = join(gbrainHome, name);
+    let mtime: string | null = null;
+    try {
+      const st = lstatSync(p);
+      mtime = st.mtime ? st.mtime.toISOString() : null;
+    } catch {
+      // stat raced a concurrent delete — still report the dir, without a date
+    }
+    const { bytes, incomplete } = walkSize(p, budget);
+    leftovers.push({ path: p, approx_bytes: bytes, size_incomplete: incomplete, dir_mtime: mtime });
+  }
+  if (leftovers.length === 0) {
+    return {
+      status: 'ok',
+      message: 'No abandoned PGLite store under the gbrain home (engine: postgres).',
+      leftovers,
+    };
+  }
+  const total = leftovers.reduce((s, l) => s + l.approx_bytes, 0);
+  const anyIncomplete = leftovers.some((l) => l.size_incomplete);
+  const listing = leftovers
+    .map(
+      (l) =>
+        `${l.path} (${l.size_incomplete ? '>=' : ''}${humanBytes(l.approx_bytes, l.size_incomplete)}` +
+        `${l.dir_mtime ? `, dir mtime ${l.dir_mtime.slice(0, 10)}` : ''})`,
+    )
+    .join('; ');
+  return {
+    status: 'warn',
+    message:
+      `Engine is postgres, but the old PGLite store remains: ${listing} — ` +
+      `${anyIncomplete ? 'at least ' : ''}${humanBytes(total, anyIncomplete)} of reclaimable disk. ` +
+      `It also inflates any backup that archives the gbrain home. Your live data is in the postgres ` +
+      `engine; once you have verified that (a recent restore or backup of it), the dir is safe to ` +
+      `delete by hand — gbrain never deletes it for you.`,
+    leftovers,
+  };
+}

test/pglite-leftovers-check.test.ts

@@ -0,0 +1,197 @@
+/**
+ * Unit tests for src/core/pglite-leftovers-check.ts (#3856).
+ *
+ * Tmp-dir fixtures only — fake store dirs with real files, no engine, no
+ * network. Mirrors npm-squat-check.test.ts's shape (#505).
+ */
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, utimesSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  assessPgliteLeftovers,
+  isMigrationLeftoverName,
+  MIGRATE_MANIFEST_NAME,
+  SIZE_WALK_MAX_ENTRIES,
+} from '../src/core/pglite-leftovers-check.ts';
+
+let root: string;
+
+/** A gbrain-home fixture; returns its path. */
+function makeHome(name: string): string {
+  const home = join(root, name);
+  mkdirSync(home, { recursive: true });
+  return home;
+}
+
+/** Lay down a fake pglite store dir with a few sized files. */
+function makeStore(home: string, dirName: string, fileBytes: number[]): string {
+  const dir = join(home, dirName);
+  mkdirSync(join(dir, 'nested'), { recursive: true });
+  fileBytes.forEach((n, i) => {
+    writeFileSync(join(dir, i === 0 ? 'base' : `nested/f${i}`), Buffer.alloc(n, 65));
+  });
+  return dir;
+}
+
+beforeAll(() => {
+  root = mkdtempSync(join(tmpdir(), 'pglite-leftovers-'));
+});
+
+afterAll(() => {
+  rmSync(root, { recursive: true, force: true });
+});
+
+describe('isMigrationLeftoverName', () => {
+  test('matches ONLY brain.pglite — the one dir the engines themselves create', () => {
+    expect(isMigrationLeftoverName('brain.pglite')).toBe(true);
+    // gbrain never creates `brain.pglite.*` siblings — no version writes a
+    // pre-migrate copy (reviewer's full-history pickaxe found no creation
+    // site), so every sibling has unknown provenance and is NOT claimed.
+    expect(isMigrationLeftoverName('brain.pglite.pre-migrate-20260524')).toBe(false);
+    expect(isMigrationLeftoverName('brain.pglite.bak')).toBe(false);
+    expect(isMigrationLeftoverName('brain.pglite2')).toBe(false);
+    expect(isMigrationLeftoverName('brain-pages')).toBe(false);
+    expect(isMigrationLeftoverName('pglite')).toBe(false);
+  });
+});
+
+describe('assessPgliteLeftovers', () => {
+  test('skips on a pglite engine — the store is live, not a leftover', () => {
+    const home = makeHome('live-pglite');
+    makeStore(home, 'brain.pglite', [1024]);
+    expect(assessPgliteLeftovers('pglite', home).status).toBe('skip');
+  });
+
+  test('skips on unknown/missing engines — only durable postgres can warn (fail open)', () => {
+    const home = makeHome('unknown-engine');
+    makeStore(home, 'brain.pglite', [1024]);
+    expect(assessPgliteLeftovers(undefined, home).status).toBe('skip');
+    expect(assessPgliteLeftovers(null, home).status).toBe('skip');
+    expect(assessPgliteLeftovers('', home).status).toBe('skip');
+    expect(assessPgliteLeftovers('supabase', home).status).toBe('skip');
+  });
+
+  test('skips when the home itself is unreadable (fail open)', () => {
+    expect(assessPgliteLeftovers('postgres', join(root, 'no-such-home')).status).toBe('skip');
+  });
+
+  test('skips while migrate-manifest.json exists — brain.pglite may be the live migration target', () => {
+    // An interrupted postgres -> pglite migration leaves the durable engine
+    // at `postgres` (config flips only on clean completion, #3194) while the
+    // manifest survives at the home root. brain.pglite is then the LIVE
+    // target: the check must not assess it, and must not advise deletion.
+    const home = makeHome('mid-migration');
+    makeStore(home, 'brain.pglite', [4096]);
+    writeFileSync(join(home, MIGRATE_MANIFEST_NAME), '{"schema_version":2}');
+    const a = assessPgliteLeftovers('postgres', home);
+    expect(a.status).toBe('skip');
+    expect(a.leftovers).toHaveLength(0);
+    expect(a.message).toContain('migration is in progress or was interrupted');
+    expect(a.message).toContain('not assessing leftovers');
+    // No deletion advice of any kind while a migration may be in flight.
+    expect(a.message).not.toContain('delete');
+    expect(a.message).not.toContain('reclaimable');
+  });
+
+  test('ok on a postgres brain with no leftover dirs', () => {
+    const home = makeHome('clean-postgres');
+    mkdirSync(join(home, 'brain-pages'));
+    makeStore(home, 'brain.pglite.bak', [512]); // out-of-scope sibling — not claimed
+    const a = assessPgliteLeftovers('postgres', home);
+    expect(a.status).toBe('ok');
+    expect(a.leftovers).toHaveLength(0);
+    expect(a.message).toContain('postgres');
+  });
+
+  test('a brain.pglite FILE (not dir) is ignored — only directories are stores', () => {
+    const home = makeHome('file-not-dir');
+    writeFileSync(join(home, 'brain.pglite'), 'not a directory');
+    expect(assessPgliteLeftovers('postgres', home).status).toBe('ok');
+  });
+
+  test('a symlinked brain.pglite is not claimed (top-level symlinks skipped)', () => {
+    const home = makeHome('symlinked-store');
+    const target = makeStore(makeHome('symlink-target-home'), 'brain.pglite', [2048]);
+    symlinkSync(target, join(home, 'brain.pglite'));
+    expect(assessPgliteLeftovers('postgres', home).status).toBe('ok');
+  });
+
+  test('warns on a postgres brain with the abandoned store — siblings stay unclaimed', () => {
+    const home = makeHome('migrated');
+    const live = makeStore(home, 'brain.pglite', [4096, 2048]);
+    // A pre-migrate-named sibling (no gbrain version creates one — unknown
+    // provenance, e.g. an operator script) must never enter the warn.
+    const pre = makeStore(home, 'brain.pglite.pre-migrate-20260524', [4096]);
+    // Freeze mtime at a known date — the in-the-wild signature (#3856).
+    const frozen = new Date('2026-05-24T02:38:42Z');
+    utimesSync(live, frozen, frozen);
+
+    const a = assessPgliteLeftovers('postgres', home);
+    expect(a.status).toBe('warn');
+    expect(a.leftovers).toHaveLength(1);
+    expect(a.leftovers[0]?.path).toBe(live);
+    expect(a.leftovers[0]?.approx_bytes).toBe(4096 + 2048);
+    expect(a.leftovers[0]?.size_incomplete).toBe(false);
+    // The message carries the receipts: path, size, an honestly-labeled
+    // dir mtime (contents can change without touching it), manual remediation.
+    expect(a.message).toContain(live);
+    expect(a.message).not.toContain(pre);
+    expect(a.message).not.toContain('pre-migrate');
+    expect(a.message).toContain('dir mtime 2026-05-24');
+    expect(a.message).not.toContain('untouched since'); // over-claim, reviewed out
+    expect(a.message).toContain('safe to delete by hand');
+    expect(a.message).toContain('backup');
+    // #3697 guard: the remediation must not invent a CLI surface.
+    expect(a.message).not.toMatch(/gbrain (cleanup|migrate cleanup|prune)/);
+  });
+
+  test('ok for a pre-migrate-named sibling alone — never claimed (unknown provenance)', () => {
+    const home = makeHome('sibling-only');
+    makeStore(home, 'brain.pglite.pre-migrate-20260101', [512]);
+    const a = assessPgliteLeftovers('postgres', home);
+    expect(a.status).toBe('ok');
+    expect(a.leftovers).toHaveLength(0);
+  });
+
+  test('nested symlinks are not followed — the walk cannot escape the store', () => {
+    const home = makeHome('nested-symlink');
+    const dir = makeStore(home, 'brain.pglite', [1024]);
+    const outside = makeHome('outside-data');
+    writeFileSync(join(outside, 'big'), Buffer.alloc(8192, 66));
+    symlinkSync(outside, join(dir, 'escape'));
+    const a = assessPgliteLeftovers('postgres', home);
+    expect(a.status).toBe('warn');
+    expect(a.leftovers[0]?.approx_bytes).toBe(1024); // the symlink target's 8 KB is NOT counted
+  });
+
+  test('an unreadable subdirectory marks the size incomplete, never exactly 0 B', () => {
+    const home = makeHome('unreadable');
+    const dir = makeStore(home, 'brain.pglite', [1024]);
+    const locked = join(dir, 'locked');
+    mkdirSync(locked);
+    writeFileSync(join(locked, 'hidden'), Buffer.alloc(4096, 67));
+    chmodSync(locked, 0o000);
+    try {
+      const a = assessPgliteLeftovers('postgres', home);
+      expect(a.status).toBe('warn');
+      expect(a.leftovers[0]?.size_incomplete).toBe(true);
+      expect(a.message).toContain('>=');
+    } finally {
+      chmodSync(locked, 0o755); // so afterAll cleanup can delete it
+    }
+  });
+
+  test('size walk is bounded by an injectable assessment-wide budget', () => {
+    const home = makeHome('bounded');
+    makeStore(home, 'brain.pglite', [10, 10, 10]); // 4 entries: base, nested/, nested/f1, nested/f2
+    // Tiny budget: the walk must stop early and mark the floor incomplete.
+    const a = assessPgliteLeftovers('postgres', home, 2);
+    expect(a.status).toBe('warn');
+    expect(a.leftovers).toHaveLength(1);
+    expect(a.leftovers[0]?.size_incomplete).toBe(true);
+    expect(a.message).toContain('at least ');
+    // Default budget is the exported constant (production callers pass nothing).
+    expect(SIZE_WALK_MAX_ENTRIES).toBe(20_000);
+  });
+});