diff --git a/scripts/envelope-to-gbrain.mjs b/scripts/envelope-to-gbrain.mjs new file mode 100644 index 000000000..84721ef1e --- /dev/null +++ b/scripts/envelope-to-gbrain.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +/** + * Import an envelope-v0 file (a JSON serialization of AI chat history; format + * spec: github.com/memvelope/memvelope) into a brain repo as one Markdown page + * per conversation, which `gbrain sync` ingests. + * + * Usage: + * node scripts/envelope-to-gbrain.mjs [outDir] + * + * Zero dependencies. Deterministic. No network. It does NOT call gbrain — it + * only writes Markdown files. + * + * Output layout: + * - One page per conversation, filename = date + conversation id (shared + * titles cannot collide; the id is the natural key). A duplicate id + * overwrites its own filename and warns on stderr; stdout reports DISTINCT + * files written, not write calls. + * - Frontmatter: `type: conversation` (keeps pages eligible for + * conversation-facts extraction and chronicle behavior after sync), the + * source provider, the conversation id, and `origin: memvelope/envelope-v0`. + * - Page `date` is the first 10 chars of the conversation's ISO-8601 + * `created_at`. Body keeps message-id citations beside each speaker turn. + * + * Memory: the whole envelope is held in memory (no streaming); envelopes are + * far smaller than the vendor exports they serialize. + * + * Verify: + * node scripts/envelope-to-gbrain.mjs test/fixtures/memvelope/sample.mve.json /tmp/out + * -> expect "wrote 1 markdown page(s)" + * bun test test/envelope-to-gbrain.test.ts + * + * STATUS: live-verified against gbrain v0.42.56.0 on 2026-07-03: the sample + * fixture -> 1 page; a real 662MB Claude export -> 353 conversations = 353 + * distinct pages (no collisions), searchable after sync with provenance and + * message-id citations intact. + */ + +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; + +const [, , envelopePath, outDir = './brain/conversations'] = process.argv; +if (!envelopePath) { + console.error('usage: node envelope-to-gbrain.mjs [outDir]'); + process.exit(1); +} + +const env = JSON.parse(readFileSync(envelopePath, 'utf8')); +if (env.memvelope !== 'envelope-v0') { + console.error(`not an envelope-v0 file (memvelope field = ${JSON.stringify(env.memvelope)})`); + process.exit(1); +} + +const slug = (s, fallback) => + (String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || fallback).slice(0, 60); + +mkdirSync(outDir, { recursive: true }); +const filesWritten = new Set(); +let collisions = 0; +const conversations = env.conversations || []; +for (const [i, c] of conversations.entries()) { + const date = (c.created_at || '').slice(0, 10); + // Name the file by the conversation's own id — the natural unique key — so two + // conversations that share a date and title can never silently overwrite each + // other. The date only leads as a human/chronological sort prefix; the id + // carries uniqueness. Positional fallback keeps names unique and deterministic + // when an envelope omits an id. + const convId = (typeof c.id === 'string' && c.id.trim()) ? c.id.trim() : `conv-${i + 1}`; + const name = `${date || '0000-00-00'}-${slug(convId, `conv-${i + 1}`)}.md`; + // gbrain reads YAML frontmatter + markdown body; keep provenance in frontmatter. + // Emit `type: conversation` so gbrain stores these as conversation pages rather + // than defaulting to the generic `concept`. gbrain is open-typed — it takes an + // explicit frontmatter `type` verbatim — and its conversation-aware features + // (conversation-facts extraction, the conversation_format_coverage check, + // chronicle eligibility) key off `type == 'conversation'`. + const front = [ + '---', + 'type: conversation', + `title: ${JSON.stringify(c.title || 'Untitled conversation')}`, + `date: ${date || 'null'}`, + `source: ${env.meta?.source_provider || 'unknown'}`, + `memvelope_conversation_id: ${JSON.stringify(c.id)}`, + 'origin: memvelope/envelope-v0', + '---', + '', + ].join('\n'); + const body = (c.messages || []) + .map((m) => `**${m.role === 'user' ? 'Me' : 'Assistant'}** (${m.ts || 'no timestamp'} · ${m.id}):\n\n${m.text}`) + .join('\n\n---\n\n'); + // Never lose a page silently: if two conversations still map to the same + // filename (e.g. an envelope carrying duplicate ids), warn loudly instead of + // overwriting in silence, and report the count of DISTINCT files written — not + // the number of write calls, which is what hid the old title-collision bug. + if (filesWritten.has(name)) { + collisions += 1; + console.warn(`warning: filename collision on "${name}" — conversation id ${JSON.stringify(c.id)} is not unique; overwriting the earlier page.`); + } + writeFileSync(join(outDir, name), front + `# ${c.title || 'Conversation'}\n\n` + body + '\n'); + filesWritten.add(name); +} +console.log(`wrote ${filesWritten.size} markdown page(s) to ${outDir} — point gbrain's sync at this directory.`); +if (collisions) { + console.warn(`warning: ${collisions} filename collision(s) — ${collisions} page(s) overwritten. Deduplicate conversation ids in the envelope to avoid data loss.`); +} diff --git a/test/envelope-to-gbrain.test.ts b/test/envelope-to-gbrain.test.ts new file mode 100644 index 000000000..733e683e9 --- /dev/null +++ b/test/envelope-to-gbrain.test.ts @@ -0,0 +1,159 @@ +/** + * Pins the Memvelope envelope importer contract: deterministic markdown output, + * provenance frontmatter, citation-bearing bodies, and loud collision handling. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const SCRIPT_PATH = join(import.meta.dir, '..', 'scripts', 'envelope-to-gbrain.mjs'); +const FIXTURE_PATH = join(import.meta.dir, 'fixtures', 'memvelope', 'sample.mve.json'); +const TEMP_DIRS: string[] = []; + +afterAll(() => { + for (const dir of TEMP_DIRS) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'envelope-to-gbrain-')); + TEMP_DIRS.push(dir); + return dir; +} + +async function runImporter(envelopePath: string, outDir = tempDir()) { + // The script is plain Node-compatible ESM; Bun can execute it directly in CI + // without requiring a separate node toolchain. + const proc = Bun.spawn([process.execPath, SCRIPT_PATH, envelopePath, outDir], { + stdout: 'pipe', + stderr: 'pipe', + }); + await proc.exited; + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + return { exitCode: proc.exitCode, stdout, stderr, outDir }; +} + +function markdownFiles(dir: string): string[] { + return readdirSync(dir).filter((name) => name.endsWith('.md')).sort(); +} + +function readOnlyMarkdown(dir: string): string { + const files = markdownFiles(dir); + expect(files).toHaveLength(1); + return readFileSync(join(dir, files[0]), 'utf8'); +} + +describe('envelope-to-gbrain importer', () => { + test('sample envelope writes exactly one markdown page and reports count', async () => { + const result = await runImporter(FIXTURE_PATH); + + expect(result.exitCode).toBe(0); + expect(markdownFiles(result.outDir)).toHaveLength(1); + expect(result.stdout).toContain('wrote 1 markdown page(s)'); + }); + + test('filename is keyed by conversation id with date prefix', async () => { + const result = await runImporter(FIXTURE_PATH); + + expect(result.exitCode).toBe(0); + expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-c-3f9a2b.md']); + }); + + test('frontmatter carries conversation provenance fields', async () => { + const result = await runImporter(FIXTURE_PATH); + const page = readOnlyMarkdown(result.outDir); + + expect(result.exitCode).toBe(0); + expect(page).toContain('type: conversation'); + expect(page).toContain('title: "Onboarding Checklist Draft"'); + expect(page).toContain('date: 2025-11-02'); + expect(page).toContain('source: chatgpt'); + expect(page).toContain('memvelope_conversation_id: "c-3f9a2b"'); + expect(page).toContain('origin: memvelope/envelope-v0'); + }); + + test('body carries role labels and message-id citations', async () => { + const result = await runImporter(FIXTURE_PATH); + const page = readOnlyMarkdown(result.outDir); + + expect(result.exitCode).toBe(0); + expect(page).toContain('· m1'); + expect(page).toContain('· m4'); + expect(page).toContain('**Me**'); + expect(page).toContain('**Assistant**'); + }); + + test('output is deterministic across repeated runs', async () => { + const first = await runImporter(FIXTURE_PATH); + const second = await runImporter(FIXTURE_PATH); + + expect(first.exitCode).toBe(0); + expect(second.exitCode).toBe(0); + expect(readOnlyMarkdown(first.outDir)).toBe(readOnlyMarkdown(second.outDir)); + }); + + test('duplicate conversation ids warn and report distinct files written', async () => { + const inputDir = tempDir(); + const envelopePath = join(inputDir, 'duplicate.mve.json'); + writeFileSync(envelopePath, JSON.stringify({ + memvelope: 'envelope-v0', + meta: { source_provider: 'chatgpt' }, + conversations: [ + { + id: 'c-repeat', + title: 'First repeated id', + created_at: '2025-11-02T14:22:51.000Z', + messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example noted the first checklist draft.' }], + }, + { + id: 'c-repeat', + title: 'Second repeated id', + created_at: '2025-11-02T15:22:51.000Z', + messages: [{ id: 'm2', role: 'assistant', ts: '2025-11-02T15:22:51.000Z', text: 'Assistant noted the repeated id collision.' }], + }, + ], + })); + + const result = await runImporter(envelopePath); + + expect(result.exitCode).toBe(0); + expect(result.stderr).toContain('warning: filename collision on "2025-11-02-c-repeat.md"'); + expect(result.stdout).toContain('wrote 1 markdown page(s)'); + expect(markdownFiles(result.outDir)).toHaveLength(1); + }); + + test('missing or foreign format is rejected', async () => { + const inputDir = tempDir(); + const envelopePath = join(inputDir, 'not-envelope.json'); + writeFileSync(envelopePath, JSON.stringify({ conversations: [] })); + + const result = await runImporter(envelopePath); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain('envelope-v0'); + }); + + test('missing conversation id uses positional fallback filename', async () => { + const inputDir = tempDir(); + const envelopePath = join(inputDir, 'missing-id.mve.json'); + writeFileSync(envelopePath, JSON.stringify({ + memvelope: 'envelope-v0', + meta: { source_provider: 'chatgpt' }, + conversations: [ + { + title: 'Missing id example', + created_at: '2025-11-02T14:22:51.000Z', + messages: [{ id: 'm1', role: 'user', ts: '2025-11-02T14:22:51.000Z', text: 'alice-example asked for a fallback filename.' }], + }, + ], + })); + + const result = await runImporter(envelopePath); + + expect(result.exitCode).toBe(0); + expect(markdownFiles(result.outDir)).toEqual(['2025-11-02-conv-1.md']); + }); +}); diff --git a/test/fixtures/memvelope/sample.mve.json b/test/fixtures/memvelope/sample.mve.json new file mode 100644 index 000000000..702f40a1c --- /dev/null +++ b/test/fixtures/memvelope/sample.mve.json @@ -0,0 +1,42 @@ +{ + "memvelope": "envelope-v0", + "meta": { + "source_provider": "chatgpt", + "conversation_count": 1, + "message_count": 4 + }, + "conversations": [ + { + "id": "c-3f9a2b", + "title": "Onboarding Checklist Draft", + "created_at": "2025-11-02T14:22:51.000Z", + "updated_at": "2025-11-02T14:31:12.000Z", + "messages": [ + { + "id": "m1", + "role": "user", + "ts": "2025-11-02T14:22:51.000Z", + "text": "alice-example is drafting acme-example's widget-co onboarding checklist and wants a concise first pass." + }, + { + "id": "m2", + "role": "assistant", + "ts": "2025-11-02T14:24:03.000Z", + "text": "Start with account setup, workspace access, sample widget review, and a first-week check-in with the acme-example owner." + }, + { + "id": "m3", + "role": "user", + "ts": "2025-11-02T14:28:19.000Z", + "text": "Add a note that bob-example should compare fund-a and fund-b reporting needs before the kickoff." + }, + { + "id": "m4", + "role": "assistant", + "ts": "2025-11-02T14:31:12.000Z", + "text": "Include a pre-kickoff step for bob-example to list fund-a and fund-b reporting questions, then confirm owners with charlie-example." + } + ] + } + ] +}