mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 18:02:30 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a42352081c | ||
|
|
070f4c5678 | ||
|
|
4b2df89935 |
@@ -1743,7 +1743,15 @@ async function extractStaleFromDB(
|
||||
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
|
||||
// µs-precision DB updated_at stayed strictly greater and the page never
|
||||
// cleared on Postgres. Stamping the exact value makes them equal.
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
|
||||
//
|
||||
// Version-arm floor: a page last edited BEFORE LINK_EXTRACTOR_VERSION_TS
|
||||
// would otherwise be stamped below the version watermark and stay
|
||||
// permanently stale (`links_extracted_at < versionTs` re-fires every run).
|
||||
// Stamp max(updated_at, versionTs) — versionTs is always a past release
|
||||
// date, so a concurrent edit's now() still exceeds the stamp and D4 holds.
|
||||
// Tie at ms precision picks updated_at_iso (its µs ≥ versionTs's .000000).
|
||||
const stampTs = new Date(page.updated_at_iso) >= new Date(versionTs) ? page.updated_at_iso : versionTs;
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampTs });
|
||||
}
|
||||
|
||||
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
|
||||
|
||||
@@ -3,11 +3,7 @@
|
||||
*
|
||||
* Usage:
|
||||
* gbrain migrate --to supabase [--url <connection_string>]
|
||||
* (--url is persisted to config.json, mode 0600, so the migrated brain
|
||||
* works without env — #1271)
|
||||
* gbrain migrate --to pglite [--path <db_path>]
|
||||
* (an explicit --path destination is bootstrapped with its own
|
||||
* <path>/.gbrain/config.json so GBRAIN_HOME=<path> just works — #1271)
|
||||
* gbrain migrate --to <engine> --force (overwrite non-empty target)
|
||||
*/
|
||||
|
||||
@@ -15,9 +11,9 @@ import { createEngine } from '../core/engine-factory.ts';
|
||||
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EngineConfig } from '../core/types.ts';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync, mkdirSync, chmodSync } from 'fs';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { resolve, join } from 'path';
|
||||
import { resolve } from 'path';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
|
||||
@@ -63,31 +59,6 @@ export interface MigrateManifest {
|
||||
started_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1271 Finding 1: make an explicit `--to pglite --path P` destination usable
|
||||
* as a standalone brain. Writes `P/.gbrain/config.json` (mode 0600, plus a
|
||||
* `*` .gitignore) so `GBRAIN_HOME=P` resolves without a manual `gbrain init`.
|
||||
* Never clobbers an existing config at the destination. Returns the written
|
||||
* config path, or null when skipped.
|
||||
*/
|
||||
export function bootstrapDestinationConfig(dbPath: string): string | null {
|
||||
const abs = resolve(dbPath);
|
||||
const dir = join(abs, '.gbrain');
|
||||
const file = join(dir, 'config.json');
|
||||
if (existsSync(file)) return null;
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const cfg: GBrainConfig = { engine: 'pglite', database_path: abs };
|
||||
writeFileSync(file, JSON.stringify(cfg, null, 2) + '\n', { mode: 0o600 });
|
||||
try { chmodSync(file, 0o600); } catch { /* platform-specific */ }
|
||||
// Same worktree-safety pattern as saveConfig()'s ensureGitignore, scoped
|
||||
// to the destination home. Don't clobber a user-customized .gitignore.
|
||||
const gitignore = join(dir, '.gitignore');
|
||||
if (!existsSync(gitignore)) {
|
||||
writeFileSync(gitignore, '*\n', { mode: 0o600 });
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
export function migrationTargetId(config: EngineConfig): string {
|
||||
const locator = config.engine === 'postgres'
|
||||
? config.database_url ?? ''
|
||||
@@ -381,25 +352,6 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
|
||||
// #1271 Finding 2 (by design, but say it out loud): the connection string
|
||||
// is persisted so the migrated brain works without env. Mode 0600.
|
||||
if (opts.targetEngine === 'postgres' && opts.targetUrl) {
|
||||
console.error('Note: the --url connection string (including credentials) is persisted to config.json (mode 0600).');
|
||||
}
|
||||
|
||||
// #1271 Finding 1: an explicit --path destination doubles as a standalone
|
||||
// GBRAIN_HOME. Best-effort — never fail a completed migration over it.
|
||||
if (opts.targetEngine === 'pglite' && opts.targetPath) {
|
||||
try {
|
||||
const written = bootstrapDestinationConfig(opts.targetPath);
|
||||
if (written) {
|
||||
console.log(`Destination bootstrapped: ${written} (usable via GBRAIN_HOME=${resolve(opts.targetPath)})`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(` WARN could not bootstrap destination config: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
clearManifest();
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ import { ensureWellFormed } from './text-safe.ts';
|
||||
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
|
||||
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
|
||||
*/
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-21T00:00:00Z';
|
||||
|
||||
// ─── Entity references ──────────────────────────────────────────
|
||||
|
||||
@@ -80,10 +80,10 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
|
||||
* Directory prefix whitelist. These are the top-level slug dirs the extractor
|
||||
* recognizes as entity references. Upstream canonical + our extensions:
|
||||
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects
|
||||
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
|
||||
* - Our domain extensions: tech, finance, personal, openclaw, ops (domain-organized wikis)
|
||||
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
|
||||
*/
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|ops)';
|
||||
|
||||
/**
|
||||
* Match `[Name](path)` markdown links pointing to entity directories.
|
||||
@@ -865,7 +865,16 @@ export function queryBasenameIndex(idx: Map<string, string[]>, name: string): st
|
||||
if (!name || typeof name !== 'string') return [];
|
||||
const trimmed = name.trim();
|
||||
if (!trimmed) return [];
|
||||
const hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
|
||||
let hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
|
||||
// Issue #2576 bug 2: path-style refs (`runbooks/2026-05-01-x`) from dirs
|
||||
// outside DIR_PATTERN reach here, but normalizeBasename strips slashes
|
||||
// into a garbage key (`runbooks2026-05-01-x`) that can never hit the
|
||||
// tail-keyed index. Fall back to the path tail so qualified refs resolve
|
||||
// by basename like everything else.
|
||||
if (!hit && trimmed.includes('/')) {
|
||||
const tail = trimmed.slice(trimmed.lastIndexOf('/') + 1).trim();
|
||||
if (tail) hit = idx.get(tail) ?? idx.get(tail.toLowerCase()) ?? idx.get(normalizeBasename(tail));
|
||||
}
|
||||
return hit ? [...hit].sort(basenameSort) : [];
|
||||
}
|
||||
|
||||
|
||||
@@ -209,6 +209,24 @@ describe('gbrain extract --stale', () => {
|
||||
expect(usRows[0]?.eq).toBe(true);
|
||||
});
|
||||
|
||||
test('version-arm floor: page edited BEFORE LINK_EXTRACTOR_VERSION_TS clears after --stale (issue #2576 bug 3)', async () => {
|
||||
// A page whose updated_at predates the version watermark used to be
|
||||
// stamped at its updated_at (< versionTs), so the version arm re-fired
|
||||
// every run — permanently stale. The sweep now floors the stamp at
|
||||
// versionTs. (The #1768 test above also covers this since the v0.42.x
|
||||
// VERSION_TS bump moved its date below the watermark, but this pins the
|
||||
// behavior explicitly so a date "repair" there can't drop coverage.)
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2000-01-01T00:00:00Z' WHERE slug = 'people/alice'`);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1);
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
// Pre-floor this stayed 1 forever (stamp < versionTs → version arm re-fires).
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
await runExtract(engine, ['--stale']);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
|
||||
|
||||
@@ -140,6 +140,15 @@ describe('extractEntityRefs', () => {
|
||||
expect(wikiRefs[0].needsResolution).toBe(true);
|
||||
});
|
||||
|
||||
test('recognizes ops/ qualified wikilinks (issue #2576 bug 2)', () => {
|
||||
// `ops` was missing from DIR_PATTERN, so [[ops/...]] fell through to
|
||||
// the generic 2c pass (needsResolution) instead of being a real ref.
|
||||
const refs = extractEntityRefs('Deployed via [[ops/services/pointer-agent]].');
|
||||
expect(refs.length).toBe(1);
|
||||
expect(refs[0].slug).toBe('ops/services/pointer-agent');
|
||||
expect(refs[0].needsResolution).toBeUndefined();
|
||||
});
|
||||
|
||||
test('skips qualified-syntax tokens (those belong to 2a)', () => {
|
||||
// [[wiki:topics/ai]] looks like 2a's qualified shape — even though
|
||||
// it wouldn't satisfy DIR_PATTERN, 2c must not claim it either
|
||||
@@ -1069,6 +1078,15 @@ describe('makeResolver — fallback chain', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test('resolveBasenameMatches: path-style ref falls back to the tail (issue #2576 bug 2)', async () => {
|
||||
// normalizeBasename strips slashes, so `runbooks/2026-05-01-pointer-agent`
|
||||
// used to normalize to a garbage key that never hit the tail-keyed index.
|
||||
const engine = makeFakeEngineWithSlugs(['ops/changes/2026-05-01-pointer-agent']);
|
||||
const r = makeResolver(engine);
|
||||
expect(await r.resolveBasenameMatches!('runbooks/2026-05-01-pointer-agent'))
|
||||
.toEqual(['ops/changes/2026-05-01-pointer-agent']);
|
||||
});
|
||||
|
||||
test('resolveBasenameMatches: case-insensitive fallback', async () => {
|
||||
const engine = makeFakeEngineWithSlugs(['companies/fast-weigh']);
|
||||
const r = makeResolver(engine);
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join, resolve } from 'path';
|
||||
import { bootstrapDestinationConfig } from '../src/commands/migrate-engine.ts';
|
||||
import { loadConfigFileOnly } from '../src/core/config.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
describe('migrate --to pglite destination bootstrap (#1271)', () => {
|
||||
test('writes <path>/.gbrain/config.json so GBRAIN_HOME=<path> resolves a brain', async () => {
|
||||
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
|
||||
const written = bootstrapDestinationConfig(dest);
|
||||
const file = join(dest, '.gbrain', 'config.json');
|
||||
expect(written).toBe(file);
|
||||
|
||||
const cfg = JSON.parse(readFileSync(file, 'utf-8'));
|
||||
expect(cfg.engine).toBe('pglite');
|
||||
expect(cfg.database_path).toBe(resolve(dest));
|
||||
expect(statSync(file).mode & 0o777).toBe(0o600);
|
||||
// worktree safety: destination home is git-ignored like saveConfig()'s home
|
||||
expect(readFileSync(join(dest, '.gbrain', '.gitignore'), 'utf-8')).toBe('*\n');
|
||||
|
||||
// The exact failure mode from #1271: config resolution under
|
||||
// GBRAIN_HOME=<path> used to find nothing ("No brain configured").
|
||||
await withEnv({ GBRAIN_HOME: dest }, () => {
|
||||
const loaded = loadConfigFileOnly();
|
||||
expect(loaded?.engine).toBe('pglite');
|
||||
expect(loaded?.database_path).toBe(resolve(dest));
|
||||
});
|
||||
});
|
||||
|
||||
test('never clobbers an existing destination config', () => {
|
||||
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
|
||||
mkdirSync(join(dest, '.gbrain'), { recursive: true });
|
||||
writeFileSync(join(dest, '.gbrain', 'config.json'), '{"engine":"postgres"}\n');
|
||||
|
||||
expect(bootstrapDestinationConfig(dest)).toBe(null);
|
||||
expect(JSON.parse(readFileSync(join(dest, '.gbrain', 'config.json'), 'utf-8')).engine).toBe('postgres');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user