Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 a42352081c test(extract): pin the version-arm stamp floor with an explicit regression test
The floor in extractStaleFromDB was only covered incidentally — the #1768
test's updated_at (2026-06-02) fell below the bumped VERSION_TS, but its
comment still says the date was chosen to sit ABOVE the watermark. A future
date 'repair' there would silently drop floor coverage. This test pins it
directly: pre-watermark page clears after --stale and stays cleared.
Verified fail-without-fix against master's extract.ts.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 11:59:09 -07:00
Garry TanandClaude Fable 5 070f4c5678 fix(extract): floor --stale stamp at LINK_EXTRACTOR_VERSION_TS so version bumps don't create permanently-stale pages
The version-ts bump in this PR exposed a latent conflict: extractStaleFromDB
stamps links_extracted_at = the page's read updated_at (#1768 µs fix), but a
page last edited BEFORE the new LINK_EXTRACTOR_VERSION_TS then lands below the
version watermark and the 'links_extracted_at < versionTs' arm re-flags it
stale on every run — an infinite re-extraction loop. Stamp
max(updated_at, versionTs); versionTs is always a past release date, so the
D4 concurrent-edit race guard still holds.

Fixes the test/extract-stale.test.ts CI failure on this branch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 10:56:38 -07:00
Garry TanandClaude Fable 5 4b2df89935 fix(extract): resolve path-style wikilinks outside DIR_PATTERN + add ops/ to whitelist (#2576 bug 2)
- Add ops to DIR_PATTERN so [[ops/services/...]] wikilinks and bare
  ops/... slug refs are recognized as qualified entity references.
- queryBasenameIndex (the shared basename matcher behind
  resolveBasenameMatches, the FS resolver, and the doctor check) now
  falls back to the path tail when a path-style ref misses — before,
  normalizeBasename stripped slashes into a garbage key that could
  never hit the tail-keyed index.
- Bump LINK_EXTRACTOR_VERSION_TS so extract --stale re-sweeps
  previously-stamped pages with the new extraction logic.

Bugs 1+3 of #2576 are covered by PR #2717 (--stale nullResolver).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:33:45 -07:00
6 changed files with 60 additions and 95 deletions
+9 -1
View File
@@ -1743,7 +1743,15 @@ async function extractStaleFromDB(
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
// µs-precision DB updated_at stayed strictly greater and the page never
// cleared on Postgres. Stamping the exact value makes them equal.
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
//
// Version-arm floor: a page last edited BEFORE LINK_EXTRACTOR_VERSION_TS
// would otherwise be stamped below the version watermark and stay
// permanently stale (`links_extracted_at < versionTs` re-fires every run).
// Stamp max(updated_at, versionTs) — versionTs is always a past release
// date, so a concurrent edit's now() still exceeds the stamp and D4 holds.
// Tie at ms precision picks updated_at_iso (its µs ≥ versionTs's .000000).
const stampTs = new Date(page.updated_at_iso) >= new Date(versionTs) ? page.updated_at_iso : versionTs;
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampTs });
}
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
+2 -50
View File
@@ -3,11 +3,7 @@
*
* Usage:
* gbrain migrate --to supabase [--url <connection_string>]
* (--url is persisted to config.json, mode 0600, so the migrated brain
* works without env — #1271)
* gbrain migrate --to pglite [--path <db_path>]
* (an explicit --path destination is bootstrapped with its own
* <path>/.gbrain/config.json so GBRAIN_HOME=<path> just works — #1271)
* gbrain migrate --to <engine> --force (overwrite non-empty target)
*/
@@ -15,9 +11,9 @@ import { createEngine } from '../core/engine-factory.ts';
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
import type { BrainEngine } from '../core/engine.ts';
import type { EngineConfig } from '../core/types.ts';
import { writeFileSync, readFileSync, existsSync, unlinkSync, mkdirSync, chmodSync } from 'fs';
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
import { createHash } from 'crypto';
import { resolve, join } from 'path';
import { resolve } from 'path';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
@@ -63,31 +59,6 @@ export interface MigrateManifest {
started_at: string;
}
/**
* #1271 Finding 1: make an explicit `--to pglite --path P` destination usable
* as a standalone brain. Writes `P/.gbrain/config.json` (mode 0600, plus a
* `*` .gitignore) so `GBRAIN_HOME=P` resolves without a manual `gbrain init`.
* Never clobbers an existing config at the destination. Returns the written
* config path, or null when skipped.
*/
export function bootstrapDestinationConfig(dbPath: string): string | null {
const abs = resolve(dbPath);
const dir = join(abs, '.gbrain');
const file = join(dir, 'config.json');
if (existsSync(file)) return null;
mkdirSync(dir, { recursive: true });
const cfg: GBrainConfig = { engine: 'pglite', database_path: abs };
writeFileSync(file, JSON.stringify(cfg, null, 2) + '\n', { mode: 0o600 });
try { chmodSync(file, 0o600); } catch { /* platform-specific */ }
// Same worktree-safety pattern as saveConfig()'s ensureGitignore, scoped
// to the destination home. Don't clobber a user-customized .gitignore.
const gitignore = join(dir, '.gitignore');
if (!existsSync(gitignore)) {
writeFileSync(gitignore, '*\n', { mode: 0o600 });
}
return file;
}
export function migrationTargetId(config: EngineConfig): string {
const locator = config.engine === 'postgres'
? config.database_url ?? ''
@@ -381,25 +352,6 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
};
saveConfig(newConfig);
// #1271 Finding 2 (by design, but say it out loud): the connection string
// is persisted so the migrated brain works without env. Mode 0600.
if (opts.targetEngine === 'postgres' && opts.targetUrl) {
console.error('Note: the --url connection string (including credentials) is persisted to config.json (mode 0600).');
}
// #1271 Finding 1: an explicit --path destination doubles as a standalone
// GBRAIN_HOME. Best-effort — never fail a completed migration over it.
if (opts.targetEngine === 'pglite' && opts.targetPath) {
try {
const written = bootstrapDestinationConfig(opts.targetPath);
if (written) {
console.log(`Destination bootstrapped: ${written} (usable via GBRAIN_HOME=${resolve(opts.targetPath)})`);
}
} catch (e) {
console.warn(` WARN could not bootstrap destination config: ${e instanceof Error ? e.message : String(e)}`);
}
}
// Clean up
clearManifest();
+13 -4
View File
@@ -28,7 +28,7 @@ import { ensureWellFormed } from './text-safe.ts';
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
*/
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-21T00:00:00Z';
// ─── Entity references ──────────────────────────────────────────
@@ -80,10 +80,10 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
* Directory prefix whitelist. These are the top-level slug dirs the extractor
* recognizes as entity references. Upstream canonical + our extensions:
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
* - Our domain extensions: tech, finance, personal, openclaw, ops (domain-organized wikis)
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
*/
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|ops)';
/**
* Match `[Name](path)` markdown links pointing to entity directories.
@@ -865,7 +865,16 @@ export function queryBasenameIndex(idx: Map<string, string[]>, name: string): st
if (!name || typeof name !== 'string') return [];
const trimmed = name.trim();
if (!trimmed) return [];
const hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
let hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
// Issue #2576 bug 2: path-style refs (`runbooks/2026-05-01-x`) from dirs
// outside DIR_PATTERN reach here, but normalizeBasename strips slashes
// into a garbage key (`runbooks2026-05-01-x`) that can never hit the
// tail-keyed index. Fall back to the path tail so qualified refs resolve
// by basename like everything else.
if (!hit && trimmed.includes('/')) {
const tail = trimmed.slice(trimmed.lastIndexOf('/') + 1).trim();
if (tail) hit = idx.get(tail) ?? idx.get(tail.toLowerCase()) ?? idx.get(normalizeBasename(tail));
}
return hit ? [...hit].sort(basenameSort) : [];
}
+18
View File
@@ -209,6 +209,24 @@ describe('gbrain extract --stale', () => {
expect(usRows[0]?.eq).toBe(true);
});
test('version-arm floor: page edited BEFORE LINK_EXTRACTOR_VERSION_TS clears after --stale (issue #2576 bug 3)', async () => {
// A page whose updated_at predates the version watermark used to be
// stamped at its updated_at (< versionTs), so the version arm re-fired
// every run — permanently stale. The sweep now floors the stamp at
// versionTs. (The #1768 test above also covers this since the v0.42.x
// VERSION_TS bump moved its date below the watermark, but this pins the
// behavior explicitly so a date "repair" there can't drop coverage.)
await engine.putPage('people/alice', personPage('Alice'));
await engine.executeRaw(`UPDATE pages SET updated_at = '2000-01-01T00:00:00Z' WHERE slug = 'people/alice'`);
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1);
await runExtract(engine, ['--stale']);
// Pre-floor this stayed 1 forever (stamp < versionTs → version arm re-fires).
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
await runExtract(engine, ['--stale']);
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
});
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
await engine.putPage('people/alice', personPage('Alice'));
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
+18
View File
@@ -140,6 +140,15 @@ describe('extractEntityRefs', () => {
expect(wikiRefs[0].needsResolution).toBe(true);
});
test('recognizes ops/ qualified wikilinks (issue #2576 bug 2)', () => {
// `ops` was missing from DIR_PATTERN, so [[ops/...]] fell through to
// the generic 2c pass (needsResolution) instead of being a real ref.
const refs = extractEntityRefs('Deployed via [[ops/services/pointer-agent]].');
expect(refs.length).toBe(1);
expect(refs[0].slug).toBe('ops/services/pointer-agent');
expect(refs[0].needsResolution).toBeUndefined();
});
test('skips qualified-syntax tokens (those belong to 2a)', () => {
// [[wiki:topics/ai]] looks like 2a's qualified shape — even though
// it wouldn't satisfy DIR_PATTERN, 2c must not claim it either
@@ -1069,6 +1078,15 @@ describe('makeResolver — fallback chain', () => {
]);
});
test('resolveBasenameMatches: path-style ref falls back to the tail (issue #2576 bug 2)', async () => {
// normalizeBasename strips slashes, so `runbooks/2026-05-01-pointer-agent`
// used to normalize to a garbage key that never hit the tail-keyed index.
const engine = makeFakeEngineWithSlugs(['ops/changes/2026-05-01-pointer-agent']);
const r = makeResolver(engine);
expect(await r.resolveBasenameMatches!('runbooks/2026-05-01-pointer-agent'))
.toEqual(['ops/changes/2026-05-01-pointer-agent']);
});
test('resolveBasenameMatches: case-insensitive fallback', async () => {
const engine = makeFakeEngineWithSlugs(['companies/fast-weigh']);
const r = makeResolver(engine);
@@ -1,40 +0,0 @@
import { describe, expect, test } from 'bun:test';
import { mkdtempSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join, resolve } from 'path';
import { bootstrapDestinationConfig } from '../src/commands/migrate-engine.ts';
import { loadConfigFileOnly } from '../src/core/config.ts';
import { withEnv } from './helpers/with-env.ts';
describe('migrate --to pglite destination bootstrap (#1271)', () => {
test('writes <path>/.gbrain/config.json so GBRAIN_HOME=<path> resolves a brain', async () => {
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
const written = bootstrapDestinationConfig(dest);
const file = join(dest, '.gbrain', 'config.json');
expect(written).toBe(file);
const cfg = JSON.parse(readFileSync(file, 'utf-8'));
expect(cfg.engine).toBe('pglite');
expect(cfg.database_path).toBe(resolve(dest));
expect(statSync(file).mode & 0o777).toBe(0o600);
// worktree safety: destination home is git-ignored like saveConfig()'s home
expect(readFileSync(join(dest, '.gbrain', '.gitignore'), 'utf-8')).toBe('*\n');
// The exact failure mode from #1271: config resolution under
// GBRAIN_HOME=<path> used to find nothing ("No brain configured").
await withEnv({ GBRAIN_HOME: dest }, () => {
const loaded = loadConfigFileOnly();
expect(loaded?.engine).toBe('pglite');
expect(loaded?.database_path).toBe(resolve(dest));
});
});
test('never clobbers an existing destination config', () => {
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
mkdirSync(join(dest, '.gbrain'), { recursive: true });
writeFileSync(join(dest, '.gbrain', 'config.json'), '{"engine":"postgres"}\n');
expect(bootstrapDestinationConfig(dest)).toBe(null);
expect(JSON.parse(readFileSync(join(dest, '.gbrain', 'config.json'), 'utf-8')).engine).toBe('postgres');
});
});