feat(import): standalone importer seeding a brain from envelope-v0 chat-history files (#3549)

One Markdown page per conversation from an envelope-v0 file (format spec: github.com/memvelope/memvelope), written into a directory gbrain sync ingests. Zero dependencies, deterministic, no network; does not call gbrain. Filenames are date + conversation id (collision-proof natural key; duplicate ids overwrite their own file and warn on stderr). Frontmatter carries type: conversation, source provider, conversation id, and origin. Bodies keep message-id citations per speaker turn. Ships as script + test + fixture only; usage and verification steps live in the script header.
2026-08-14 08:53:22 +00:00 · 2026-07-29 11:56:20 -07:00
parent 913d2d7f79
commit 1057bf4368
3 changed files with 304 additions and 0 deletions
@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+/**
+ * Import an envelope-v0 file (a JSON serialization of AI chat history; format
+ * spec: github.com/memvelope/memvelope) into a brain repo as one Markdown page
+ * per conversation, which `gbrain sync` ingests.
+ *
+ * Usage:
+ *   node scripts/envelope-to-gbrain.mjs <envelope.mve.json> [outDir]
+ *
+ * Zero dependencies. Deterministic. No network. It does NOT call gbrain — it
+ * only writes Markdown files.
+ *
+ * Output layout:
+ *   - One page per conversation, filename = date + conversation id (shared
+ *     titles cannot collide; the id is the natural key). A duplicate id
+ *     overwrites its own filename and warns on stderr; stdout reports DISTINCT
+ *     files written, not write calls.
+ *   - Frontmatter: `type: conversation` (keeps pages eligible for
+ *     conversation-facts extraction and chronicle behavior after sync), the
+ *     source provider, the conversation id, and `origin: memvelope/envelope-v0`.
+ *   - Page `date` is the first 10 chars of the conversation's ISO-8601
+ *     `created_at`. Body keeps message-id citations beside each speaker turn.
+ *
+ * Memory: the whole envelope is held in memory (no streaming); envelopes are
+ * far smaller than the vendor exports they serialize.
+ *
+ * Verify:
+ *   node scripts/envelope-to-gbrain.mjs test/fixtures/memvelope/sample.mve.json /tmp/out
+ *     -> expect "wrote 1 markdown page(s)"
+ *   bun test test/envelope-to-gbrain.test.ts
+ *
+ * STATUS: live-verified against gbrain v0.42.56.0 on 2026-07-03: the sample
+ * fixture -> 1 page; a real 662MB Claude export -> 353 conversations = 353
+ * distinct pages (no collisions), searchable after sync with provenance and
+ * message-id citations intact.
+ */
+
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const [, , envelopePath, outDir = './brain/conversations'] = process.argv;
+if (!envelopePath) {
+  console.error('usage: node envelope-to-gbrain.mjs <envelope.mve.json> [outDir]');
+  process.exit(1);
+}
+
+const env = JSON.parse(readFileSync(envelopePath, 'utf8'));
+if (env.memvelope !== 'envelope-v0') {
+  console.error(`not an envelope-v0 file (memvelope field = ${JSON.stringify(env.memvelope)})`);
+  process.exit(1);
+}
+
+const slug = (s, fallback) =>
+  (String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback).slice(0, 60);
+
+mkdirSync(outDir, { recursive: true });
+const filesWritten = new Set();
+let collisions = 0;
+const conversations = env.conversations || [];
+for (const [i, c] of conversations.entries()) {
+  const date = (c.created_at || '').slice(0, 10);
+  // Name the file by the conversation's own id — the natural unique key — so two
+  // conversations that share a date and title can never silently overwrite each
+  // other. The date only leads as a human/chronological sort prefix; the id
+  // carries uniqueness. Positional fallback keeps names unique and deterministic
+  // when an envelope omits an id.
+  const convId = (typeof c.id === 'string' && c.id.trim()) ? c.id.trim() : `conv-${i + 1}`;
+  const name = `${date || '0000-00-00'}-${slug(convId, `conv-${i + 1}`)}.md`;
+  // gbrain reads YAML frontmatter + markdown body; keep provenance in frontmatter.
+  // Emit `type: conversation` so gbrain stores these as conversation pages rather
+  // than defaulting to the generic `concept`. gbrain is open-typed — it takes an
+  // explicit frontmatter `type` verbatim — and its conversation-aware features
+  // (conversation-facts extraction, the conversation_format_coverage check,
+  // chronicle eligibility) key off `type == 'conversation'`.
+  const front = [
+    '---',
+    'type: conversation',
+    `title: ${JSON.stringify(c.title || 'Untitled conversation')}`,
+    `date: ${date || 'null'}`,
+    `source: ${env.meta?.source_provider || 'unknown'}`,
+    `memvelope_conversation_id: ${JSON.stringify(c.id)}`,
+    'origin: memvelope/envelope-v0',
+    '---',
+    '',
+  ].join('\n');
+  const body = (c.messages || [])
+    .map((m) => `**${m.role === 'user' ? 'Me' : 'Assistant'}** (${m.ts || 'no timestamp'} · ${m.id}):\n\n${m.text}`)
+    .join('\n\n---\n\n');
+  // Never lose a page silently: if two conversations still map to the same
+  // filename (e.g. an envelope carrying duplicate ids), warn loudly instead of
+  // overwriting in silence, and report the count of DISTINCT files written — not
+  // the number of write calls, which is what hid the old title-collision bug.
+  if (filesWritten.has(name)) {
+    collisions += 1;
+    console.warn(`warning: filename collision on "${name}" — conversation id ${JSON.stringify(c.id)} is not unique; overwriting the earlier page.`);
+  }
+  writeFileSync(join(outDir, name), front + `# ${c.title || 'Conversation'}\n\n` + body + '\n');
+  filesWritten.add(name);
+}
+console.log(`wrote ${filesWritten.size} markdown page(s) to ${outDir} — point gbrain's sync at this directory.`);
+if (collisions) {
+  console.warn(`warning: ${collisions} filename collision(s) — ${collisions} page(s) overwritten. Deduplicate conversation ids in the envelope to avoid data loss.`);
+}
@@ -0,0 +1,159 @@
+/**
+ * Pins the Memvelope envelope importer contract: deterministic markdown output,
+ * provenance frontmatter, citation-bearing bodies, and loud collision handling.
+ */
+import { afterAll, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const SCRIPT_PATH = join(import.meta.dir, '..', 'scripts', 'envelope-to-gbrain.mjs');
+const FIXTURE_PATH = join(import.meta.dir, 'fixtures', 'memvelope', 'sample.mve.json');
+const TEMP_DIRS: string[] = [];
+
+afterAll(() => {
+  for (const dir of TEMP_DIRS) {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function tempDir(): string {
+  const dir = mkdtempSync(join(tmpdir(), 'envelope-to-gbrain-'));
+  TEMP_DIRS.push(dir);
+  return dir;
+}
+
+async function runImporter(envelopePath: string, outDir = tempDir()) {
+  // The script is plain Node-compatible ESM; Bun can execute it directly in CI
+  // without requiring a separate node toolchain.
+  const proc = Bun.spawn([process.execPath, SCRIPT_PATH, envelopePath, outDir], {
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+  await proc.exited;
+  const stdout = await new Response(proc.stdout).text();
+  const stderr = await new Response(proc.stderr).text();
+  return { exitCode: proc.exitCode, stdout, stderr, outDir };
+}
+
+function markdownFiles(dir: string): string[] {
+  return readdirSync(dir).filter((name) => name.endsWith('.md')).sort();
+}
+
+function readOnlyMarkdown(dir: string): string {
+  const files = markdownFiles(dir);
+  expect(files).toHaveLength(1);
+  return readFileSync(join(dir, files[0]), 'utf8');
+}
+
+describe('envelope-to-gbrain importer', () => {
+  test('sample envelope writes exactly one markdown page and reports count', async () => {
+    const result = await runImporter(FIXTURE_PATH);
+
+    expect(result.exitCode).toBe(0);
+    expect(markdownFiles(result.outDir)).toHaveLength(1);
+    expect(result.stdout).toContain('wrote 1 markdown page(s)');
+  });
+
+  test('filename is keyed by conversation id with date prefix', async () => {
+    const result = await runImporter(FIXTURE_PATH);
+
+    expect(result.exitCode).toBe(0);
+    expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-c-3f9a2b.md']);
+  });
+
+  test('frontmatter carries conversation provenance fields', async () => {
+    const result = await runImporter(FIXTURE_PATH);
+    const page = readOnlyMarkdown(result.outDir);
+
+    expect(result.exitCode).toBe(0);
+    expect(page).toContain('type: conversation');
+    expect(page).toContain('title: "Onboarding Checklist Draft"');
+    expect(page).toContain('date: 2025-11-02');
+    expect(page).toContain('source: chatgpt');
+    expect(page).toContain('memvelope_conversation_id: "c-3f9a2b"');
+    expect(page).toContain('origin: memvelope/envelope-v0');
+  });
+
+  test('body carries role labels and message-id citations', async () => {
+    const result = await runImporter(FIXTURE_PATH);
+    const page = readOnlyMarkdown(result.outDir);
+
+    expect(result.exitCode).toBe(0);
+    expect(page).toContain('· m1');
+    expect(page).toContain('· m4');
+    expect(page).toContain('**Me**');
+    expect(page).toContain('**Assistant**');
+  });
+
+  test('output is deterministic across repeated runs', async () => {
+    const first = await runImporter(FIXTURE_PATH);
+    const second = await runImporter(FIXTURE_PATH);
+
+    expect(first.exitCode).toBe(0);
+    expect(second.exitCode).toBe(0);
+    expect(readOnlyMarkdown(first.outDir)).toBe(readOnlyMarkdown(second.outDir));
+  });
+
+  test('duplicate conversation ids warn and report distinct files written', async () => {
+    const inputDir = tempDir();
+    const envelopePath = join(inputDir, 'duplicate.mve.json');
+    writeFileSync(envelopePath, JSON.stringify({
+      memvelope: 'envelope-v0',
+      meta: { source_provider: 'chatgpt' },
+      conversations: [
+        {
+          id: 'c-repeat',
+          title: 'First repeated id',
+          created_at: '2025-11-02T14:22:51.000Z',
+          messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example noted the first checklist draft.' }],
+        },
+        {
+          id: 'c-repeat',
+          title: 'Second repeated id',
+          created_at: '2025-11-02T15:22:51.000Z',
+          messages: [{ id: 'm2', role: 'assistant', ts: '2025-11-02T15:22:51.000Z', text: 'Assistant noted the repeated id collision.' }],
+        },
+      ],
+    }));
+
+    const result = await runImporter(envelopePath);
+
+    expect(result.exitCode).toBe(0);
+    expect(result.stderr).toContain('warning: filename collision on "2025-11-02-c-repeat.md"');
+    expect(result.stdout).toContain('wrote 1 markdown page(s)');
+    expect(markdownFiles(result.outDir)).toHaveLength(1);
+  });
+
+  test('missing or foreign format is rejected', async () => {
+    const inputDir = tempDir();
+    const envelopePath = join(inputDir, 'not-envelope.json');
+    writeFileSync(envelopePath, JSON.stringify({ conversations: [] }));
+
+    const result = await runImporter(envelopePath);
+
+    expect(result.exitCode).toBe(1);
+    expect(result.stderr).toContain('envelope-v0');
+  });
+
+  test('missing conversation id uses positional fallback filename', async () => {
+    const inputDir = tempDir();
+    const envelopePath = join(inputDir, 'missing-id.mve.json');
+    writeFileSync(envelopePath, JSON.stringify({
+      memvelope: 'envelope-v0',
+      meta: { source_provider: 'chatgpt' },
+      conversations: [
+        {
+          title: 'Missing id example',
+          created_at: '2025-11-02T14:22:51.000Z',
+          messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example asked for a fallback filename.' }],
+        },
+      ],
+    }));
+
+    const result = await runImporter(envelopePath);
+
+    expect(result.exitCode).toBe(0);
+    expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-conv-1.md']);
+  });
+});
@@ -0,0 +1,42 @@
+{
+  "memvelope": "envelope-v0",
+  "meta": {
+    "source_provider": "chatgpt",
+    "conversation_count": 1,
+    "message_count": 4
+  },
+  "conversations": [
+    {
+      "id": "c-3f9a2b",
+      "title": "Onboarding Checklist Draft",
+      "created_at": "2025-11-02T14:22:51.000Z",
+      "updated_at": "2025-11-02T14:31:12.000Z",
+      "messages": [
+        {
+          "id": "m1",
+          "role": "user",
+          "ts": "2025-11-02T14:22:51.000Z",
+          "text": "alice-example is drafting acme-example's widget-co onboarding checklist and wants a concise first pass."
+        },
+        {
+          "id": "m2",
+          "role": "assistant",
+          "ts": "2025-11-02T14:24:03.000Z",
+          "text": "Start with account setup, workspace access, sample widget review, and a first-week check-in with the acme-example owner."
+        },
+        {
+          "id": "m3",
+          "role": "user",
+          "ts": "2025-11-02T14:28:19.000Z",
+          "text": "Add a note that bob-example should compare fund-a and fund-b reporting needs before the kickoff."
+        },
+        {
+          "id": "m4",
+          "role": "assistant",
+          "ts": "2025-11-02T14:31:12.000Z",
+          "text": "Include a pre-kickoff step for bob-example to list fund-a and fund-b reporting questions, then confirm owners with charlie-example."
+        }
+      ]
+    }
+  ]
+}