mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 09:52:22 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
08f2397615 |
@@ -186,18 +186,6 @@ export function isSourceStale(src: SourceRow, now = Date.now(), floorMin = FULL_
|
||||
return ageMin >= floorMin;
|
||||
}
|
||||
|
||||
/**
|
||||
* #2060: count sources past the per-source cycle freshness floor. Consumed
|
||||
* by autopilot's dispatch decision — a stale source forces the fanout path
|
||||
* even when the doctor plan is small (score 70–94, plan ≤ 3, est < 300s),
|
||||
* so targeted mode can't leave cycle_freshness stale indefinitely.
|
||||
* dispatchPerSource's own throttles (skipped_fresh / fanoutMax / failure
|
||||
* cooldown) bound the resulting work.
|
||||
*/
|
||||
export function countStaleSources(sources: SourceRow[], now = Date.now(), floorMin = FULL_CYCLE_FLOOR_MIN): number {
|
||||
return sources.filter((s) => isSourceStale(s, now, floorMin)).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Most recent SUCCESSFUL cycle for a source. Prefers `last_source_cycle_at`
|
||||
* (per-source phases, written by the split cycle) and falls back to the legacy
|
||||
|
||||
@@ -901,27 +901,13 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
const FULL_CYCLE_FLOOR_MIN = 60;
|
||||
const minutesSinceLastFull = (Date.now() - lastFullCycleAt) / 60000;
|
||||
|
||||
// #2060: stale per-source cycle freshness is a dispatch input. Without
|
||||
// it, a brain sitting at score 70–94 with a small targeted plan (≤3
|
||||
// steps, <300s) stays in targeted mode indefinitely and no per-source
|
||||
// cycle is ever dispatched — cycle_freshness never advances. A stale
|
||||
// source forces the fanout path; dispatchPerSource's throttles
|
||||
// (skipped_fresh / fanoutMax / failure cooldown) bound the work.
|
||||
// Fail-open to 0: a read failure must not block dispatch.
|
||||
let staleCycleSources = 0;
|
||||
try {
|
||||
const { countStaleSources } = await import('./autopilot-fanout.ts');
|
||||
staleCycleSources = countStaleSources(await engine.listAllSources({ localPathOnly: true }));
|
||||
} catch { /* fail-open: freshness is a dispatch hint, not a gate */ }
|
||||
|
||||
const shouldFullCycle =
|
||||
(score >= 95 && plan.length === 0 && minutesSinceLastFull >= FULL_CYCLE_FLOOR_MIN) ||
|
||||
plan.length > 3 ||
|
||||
estTotal >= 300 ||
|
||||
score < 70 ||
|
||||
staleCycleSources > 0;
|
||||
score < 70;
|
||||
|
||||
const shouldSleep = score >= 95 && plan.length === 0 && minutesSinceLastFull < FULL_CYCLE_FLOOR_MIN && staleCycleSources === 0;
|
||||
const shouldSleep = score >= 95 && plan.length === 0 && minutesSinceLastFull < FULL_CYCLE_FLOOR_MIN;
|
||||
|
||||
if (shouldSleep) {
|
||||
if (jsonMode) {
|
||||
|
||||
@@ -1651,7 +1651,7 @@ async function extractTimelineFromDB(
|
||||
* make re-extraction idempotent). EVERY processed page is stamped, including
|
||||
* zero-link pages — they WERE processed.
|
||||
*/
|
||||
async function extractStaleFromDB(
|
||||
export async function extractStaleFromDB(
|
||||
engine: BrainEngine,
|
||||
opts: {
|
||||
dryRun: boolean;
|
||||
|
||||
+39
-1
@@ -1479,7 +1479,31 @@ export async function registerBuiltinHandlers(
|
||||
embedSkipReason = 'auto_embed_disabled';
|
||||
}
|
||||
|
||||
return { ...result, embed_job_id: embedJobId, embed_skip_reason: embedSkipReason };
|
||||
// #2849: large-sync extract deferral follow-up. performSync skips inline
|
||||
// link/timeline extraction when totalChanges > 100, leaving
|
||||
// links_extracted_at unstamped. A standalone sync job (webhook push,
|
||||
// sync trigger) has no autopilot extract phase behind it, so the pages
|
||||
// would stay extraction-stale until a manual `gbrain extract --stale`.
|
||||
// Queue a source-scoped stale sweep instead. Best-effort + idempotent:
|
||||
// a duplicate sweep finds 0 stale pages and no-ops.
|
||||
let extractJobId: number | null = null;
|
||||
if (result.extractDeferred) {
|
||||
try {
|
||||
const { MinionQueue } = await import('../core/minions/queue.ts');
|
||||
const queue = new MinionQueue(engine);
|
||||
const followUp = await queue.add(
|
||||
'extract',
|
||||
{ stale: true, ...(sourceId ? { sourceId } : {}) },
|
||||
{
|
||||
idempotency_key: `sync-extract-stale:${sourceId ?? 'default'}:${Math.floor(Date.now() / 30_000)}`,
|
||||
maxWaiting: 1,
|
||||
},
|
||||
);
|
||||
extractJobId = followUp.id;
|
||||
} catch { /* best-effort: extract --stale sweeps it later */ }
|
||||
}
|
||||
|
||||
return { ...result, embed_job_id: embedJobId, embed_skip_reason: embedSkipReason, extract_stale_job_id: extractJobId };
|
||||
});
|
||||
|
||||
registerBuiltinJob(worker, engine, 'embed', async (job) => {
|
||||
@@ -1652,6 +1676,20 @@ export async function registerBuiltinHandlers(
|
||||
});
|
||||
|
||||
worker.register('extract', async (job) => {
|
||||
// #2849: stale-sweep mode — the sync handler's large-sync deferral
|
||||
// follow-up. DB-source (reads page content from the DB, so it runs on
|
||||
// checkout-less brains), source-scopable, idempotent. Same core as
|
||||
// `gbrain extract --stale`.
|
||||
if (job.data.stale === true) {
|
||||
const { extractStaleFromDB } = await import('./extract.ts');
|
||||
return await extractStaleFromDB(engine, {
|
||||
dryRun: !!job.data.dryRun,
|
||||
jsonMode: false,
|
||||
includeFrontmatter: false,
|
||||
sourceIdFilter: typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined,
|
||||
catchUp: false,
|
||||
});
|
||||
}
|
||||
const { runExtractCore } = await import('./extract.ts');
|
||||
const mode = (typeof job.data.mode === 'string' && ['links', 'timeline', 'all'].includes(job.data.mode))
|
||||
? (job.data.mode as 'links' | 'timeline' | 'all')
|
||||
|
||||
@@ -2146,8 +2146,13 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Other event types (ping, pull_request, etc.) return 202 'ignored'
|
||||
// so GitHub doesn't retry.
|
||||
// D15.5: HMAC compare uses the shared safeHexEqual helper.
|
||||
// D18: submits 'sync' job with auto_embed_backfill=true and priority -10
|
||||
// (above autopilot's 0).
|
||||
// D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
|
||||
// priority -10 (above autopilot's 0). noExtract:false opts normal
|
||||
// incremental pushes into sync's inline link/timeline extraction (#2849
|
||||
// — the standalone sync handler defaults noExtract to TRUE, which left
|
||||
// webhook-imported pages permanently stale). Large (>100 file) pushes
|
||||
// defer inline extract; the sync handler queues an extract --stale
|
||||
// follow-up job for that branch.
|
||||
// ---------------------------------------------------------------------------
|
||||
const githubWebhookLimiter = rateLimit({
|
||||
windowMs: 60_000,
|
||||
@@ -2267,6 +2272,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
'sync',
|
||||
{
|
||||
sourceId: source.id,
|
||||
noExtract: false,
|
||||
auto_embed_backfill: true,
|
||||
embed_reason: 'webhook',
|
||||
},
|
||||
|
||||
+19
-1
@@ -222,6 +222,14 @@ export interface SyncResult {
|
||||
* everything," the exact misdiagnosis in the #1794 recurrence report.
|
||||
*/
|
||||
bankedFiles?: number;
|
||||
/**
|
||||
* #2849: true when extraction was REQUESTED (noExtract false) but this sync
|
||||
* skipped inline link/timeline extraction because totalChanges > 100 (the
|
||||
* #1794 large-sync deferral). links_extracted_at stays unstamped for the
|
||||
* imported pages. The standalone `sync` job handler queues a source-scoped
|
||||
* `extract --stale` follow-up when set; CLI runs print the manual hint.
|
||||
*/
|
||||
extractDeferred?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1379,6 +1387,10 @@ See also:
|
||||
{
|
||||
sourceId: sourceIdArg,
|
||||
repoPath: source.local_path,
|
||||
// #2849: opt in to inline extraction — the standalone sync handler
|
||||
// defaults noExtract to TRUE (dedupe for doctor's [sync, extract]
|
||||
// remediation plan), which would leave triggered syncs extraction-stale.
|
||||
noExtract: false,
|
||||
auto_embed_backfill: true,
|
||||
embed_reason: 'sync_trigger',
|
||||
},
|
||||
@@ -3287,11 +3299,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// the stale sweep scans the whole source, so banked-across-runs pages are
|
||||
// covered regardless.
|
||||
const extractOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
|
||||
let extractDeferred = false;
|
||||
if (!opts.noExtract && totalChanges > 100 && pagesAffected.length > 0) {
|
||||
// #2849: surface the deferral to callers. A standalone sync job (webhook
|
||||
// push, sync trigger) has no autopilot extract phase behind it, so the
|
||||
// job handler queues an `extract --stale` follow-up off this flag.
|
||||
extractDeferred = true;
|
||||
slog(
|
||||
` Large sync: deferring link/timeline extraction. ` +
|
||||
`Run 'gbrain extract --stale${opts.sourceId ? ` --source-id ${opts.sourceId}` : ''}' ` +
|
||||
`(or let the autopilot cycle's extract phase sweep it).`,
|
||||
`(sync jobs queue this follow-up automatically).`,
|
||||
);
|
||||
}
|
||||
if (!opts.noExtract && totalChanges <= 100 && pagesAffected.length > 0) {
|
||||
@@ -3400,6 +3417,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
chunksCreated,
|
||||
embedded,
|
||||
pagesAffected,
|
||||
extractDeferred,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+10
-19
@@ -854,10 +854,7 @@ interface SyncPhaseResult extends PhaseResult {
|
||||
/**
|
||||
* Resolve the source id for a brain directory by looking up the sources
|
||||
* table. Returns undefined when no registered source matches (falls back
|
||||
* to pre-v0.18 global config.sync.* keys) OR when MORE than one source
|
||||
* claims the path — an ambiguous match must not scope phases or stamp
|
||||
* last_full_cycle_at for an arbitrarily-picked source (the "freshness
|
||||
* stamp that lies" this resolution exists to prevent).
|
||||
* to pre-v0.18 global config.sync.* keys).
|
||||
*/
|
||||
async function resolveSourceForDir(
|
||||
engine: BrainEngine,
|
||||
@@ -868,10 +865,10 @@ async function resolveSourceForDir(
|
||||
if (brainDir === null) return undefined;
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM sources WHERE local_path = $1 LIMIT 2`,
|
||||
`SELECT id FROM sources WHERE local_path = $1 LIMIT 1`,
|
||||
[brainDir],
|
||||
);
|
||||
return rows.length === 1 ? rows[0]!.id : undefined;
|
||||
return rows[0]?.id;
|
||||
} catch {
|
||||
// sources table might not exist on very old brains — fall through.
|
||||
return undefined;
|
||||
@@ -2368,23 +2365,17 @@ export async function runCycle(
|
||||
}
|
||||
|
||||
// v0.38 (codex r1 P0-5): persist per-source cycle completion timestamp
|
||||
// when the cycle ran successfully against a resolvable source. Read by
|
||||
// autopilot's per-source freshness gate next tick.
|
||||
//
|
||||
// #1993: keyed off `cycleSourceId` (opts.sourceId ?? the source resolved
|
||||
// from brainDir) — the SAME id the cycle locked + scoped its phases to —
|
||||
// NOT raw opts.sourceId. The autopilot's inline cycle sets brainDir but
|
||||
// passes no explicit sourceId, so keying off opts.sourceId alone never
|
||||
// advanced last_full_cycle_at and cycle_freshness stayed stale even while
|
||||
// the autopilot cycled every interval. Skipped when:
|
||||
// - no source resolves (engine null, or no checkout AND no opts.sourceId)
|
||||
// when the cycle ran successfully against an explicit source. Read by
|
||||
// autopilot's per-source freshness gate next tick. Skipped when:
|
||||
// - opts.sourceId is unset (legacy callers — autopilot still here)
|
||||
// - engine is null (no-DB path)
|
||||
// - status is 'failed' or 'skipped' (don't mark a non-run as fresh)
|
||||
// - dryRun (writes are out of scope)
|
||||
//
|
||||
// Best-effort: a write failure does NOT change the CycleReport status.
|
||||
// The cost of writing the wrong timestamp post-failure is higher than
|
||||
// the cost of missing a successful write (next cycle will redo work).
|
||||
if (cycleSourceId && engine && !dryRun && !aborted && (status === 'ok' || status === 'clean' || status === 'partial')) {
|
||||
if (opts.sourceId && engine && !dryRun && !aborted && (status === 'ok' || status === 'clean' || status === 'partial')) {
|
||||
try {
|
||||
const nowIso = new Date().toISOString();
|
||||
// #2194 fix #3 (the cycle split): `last_source_cycle_at` is the NEW gate
|
||||
@@ -2394,13 +2385,13 @@ export async function runCycle(
|
||||
// phases (those gate on autopilot.last_global_at), so writing it on a
|
||||
// source-only cycle does not re-introduce the freshness poisoning codex
|
||||
// flagged in the rejected skip-based design.
|
||||
await engine.updateSourceConfig(cycleSourceId, {
|
||||
await engine.updateSourceConfig(opts.sourceId, {
|
||||
last_source_cycle_at: nowIso,
|
||||
last_full_cycle_at: nowIso,
|
||||
});
|
||||
} catch (e) {
|
||||
// Best-effort; cycle already succeeded by the time we get here.
|
||||
console.warn(`[cycle] failed to write last_source_cycle_at for source ${cycleSourceId}: ${e instanceof Error ? e.message : String(e)}`);
|
||||
console.warn(`[cycle] failed to write last_source_cycle_at for source ${opts.sourceId}: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,20 +54,6 @@ describe('autopilot.ts ↔ dispatchPerSource wiring', () => {
|
||||
expect(AUTOPILOT_SRC).toMatch(/lastFullCycleAt\s*=\s*Date\.now\(\)/);
|
||||
});
|
||||
|
||||
test('stale per-source cycle freshness is a shouldFullCycle input (#2060)', () => {
|
||||
// Targeted mode (score 70–94, plan ≤3, est <300s) must not be able to
|
||||
// starve per-source cycle dispatch: a stale source (per countStaleSources
|
||||
// over listAllSources) forces the fanout path, and the sleep gate must
|
||||
// not fire while stale sources exist. Without these terms, cycle
|
||||
// freshness never advances for a brain that always lands in targeted mode.
|
||||
expect(AUTOPILOT_SRC).toMatch(/countStaleSources/);
|
||||
const fullCycleDeclIdx = AUTOPILOT_SRC.indexOf('const shouldFullCycle');
|
||||
expect(fullCycleDeclIdx).toBeGreaterThan(-1);
|
||||
const decl = AUTOPILOT_SRC.slice(fullCycleDeclIdx, fullCycleDeclIdx + 700);
|
||||
expect(decl).toMatch(/staleCycleSources\s*>\s*0/);
|
||||
expect(decl).toMatch(/const shouldSleep[^;]*staleCycleSources\s*===\s*0/);
|
||||
});
|
||||
|
||||
test('does NOT regress to the single-job dispatch on the full-cycle path', () => {
|
||||
// Pre-PR: the shouldFullCycle branch did:
|
||||
// const job = await queue.add('autopilot-cycle', { repoPath }, {
|
||||
|
||||
@@ -14,7 +14,6 @@ import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
readLastFullCycleAt,
|
||||
isSourceStale,
|
||||
countStaleSources,
|
||||
selectSourcesForDispatch,
|
||||
resolveFanoutMax,
|
||||
dispatchPerSource,
|
||||
@@ -75,23 +74,6 @@ describe('isSourceStale', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('countStaleSources (#2060 dispatch-decision input)', () => {
|
||||
const NOW = Date.parse('2026-05-22T12:00:00.000Z');
|
||||
test('counts never-cycled + past-floor sources, ignores fresh', () => {
|
||||
const sources = [
|
||||
src('never-cycled'), // stale (null)
|
||||
src('old', new Date(NOW - 2 * 60 * 60_000).toISOString()), // stale (2h)
|
||||
src('fresh', new Date(NOW - 30 * 60_000).toISOString()), // fresh (30min)
|
||||
];
|
||||
expect(countStaleSources(sources, NOW)).toBe(2);
|
||||
});
|
||||
test('returns 0 for all-fresh and for empty list', () => {
|
||||
const fresh = src('a', new Date(NOW - 10 * 60_000).toISOString());
|
||||
expect(countStaleSources([fresh], NOW)).toBe(0);
|
||||
expect(countStaleSources([], NOW)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectSourcesForDispatch', () => {
|
||||
const NOW = Date.parse('2026-05-22T12:00:00.000Z');
|
||||
const fresh = (id: string, agoMin: number) =>
|
||||
|
||||
@@ -3,10 +3,8 @@
|
||||
* cycles. Closes codex round-1 P0-5 (write site for last_full_cycle_at
|
||||
* was unspecified pre-PR).
|
||||
*
|
||||
* Conditions for write (keyed off `cycleSourceId` = opts.sourceId ?? the
|
||||
* source resolved from brainDir, so the autopilot's inline cycle — brainDir
|
||||
* set, no explicit sourceId — also advances the timestamp, #1993):
|
||||
* - a source resolves (explicit sourceId, or brainDir matches a source)
|
||||
* Conditions for write:
|
||||
* - opts.sourceId is set (legacy callers without sourceId skip the write)
|
||||
* - engine is non-null (no-DB path skips)
|
||||
* - status is 'ok' | 'clean' | 'partial' (failed/skipped don't mark fresh)
|
||||
* - dryRun is false
|
||||
@@ -92,45 +90,17 @@ describe('runCycle last_full_cycle_at exit hook', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('no explicit sourceId but brainDir resolves a source → writes the resolved source timestamp', async () => {
|
||||
test('legacy caller (no sourceId) does NOT write any source timestamp', async () => {
|
||||
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
||||
// The autopilot's inline cycle sets brainDir but passes no sourceId.
|
||||
// runCycle resolves the source from brainDir (local_path match) into
|
||||
// cycleSourceId and stamps last_full_cycle_at for it — otherwise
|
||||
// cycle_freshness reports the brain stale even while the autopilot
|
||||
// cycles every interval (#1993).
|
||||
await seedSource('resolved-from-dir'); // local_path = brainDir
|
||||
expect(await readLastFullCycleAt('resolved-from-dir')).toBeNull();
|
||||
|
||||
const t0 = Date.now();
|
||||
const report = await runCycle(engine, {
|
||||
brainDir,
|
||||
phases: ['lint'],
|
||||
});
|
||||
expect(['ok', 'clean']).toContain(report.status);
|
||||
|
||||
const after = await readLastFullCycleAt('resolved-from-dir');
|
||||
expect(after).not.toBeNull();
|
||||
expect(new Date(after!).getTime()).toBeGreaterThanOrEqual(t0);
|
||||
});
|
||||
});
|
||||
|
||||
test('no sourceId and brainDir matches no source → does not write', async () => {
|
||||
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
||||
// A source exists but its local_path does NOT match brainDir, so
|
||||
// resolveSourceForDir returns undefined, cycleSourceId is undefined,
|
||||
// and no per-source timestamp is written.
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config, archived, created_at)
|
||||
VALUES ('unmatched', 'unmatched', '/no/such/repo', '{}'::jsonb, false, NOW())
|
||||
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
|
||||
[],
|
||||
);
|
||||
await seedSource('default-like');
|
||||
// No sourceId passed; should remain untouched.
|
||||
await runCycle(engine, {
|
||||
brainDir,
|
||||
phases: ['lint'],
|
||||
});
|
||||
expect(await readLastFullCycleAt('unmatched')).toBeNull();
|
||||
// No per-source write happens; default source's config stays empty.
|
||||
const after = await readLastFullCycleAt('default-like');
|
||||
expect(after).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { createHmac } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { safeHexEqual } from '../src/core/timing-safe.ts';
|
||||
|
||||
const GITHUB_SECRET = 'super-secret-webhook-key';
|
||||
@@ -123,3 +124,25 @@ describe('Branch ref construction (D5)', () => {
|
||||
expect(pushedRef === `refs/heads/${trackedBranch}`).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Webhook sync job extraction contract (#2849)', () => {
|
||||
test('opts into extraction before the pushed commit is consumed', () => {
|
||||
const serveSource = readFileSync(
|
||||
new URL('../src/commands/serve-http.ts', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const routeStart = serveSource.indexOf("'/webhooks/github'");
|
||||
const queueStart = serveSource.indexOf('const job = await queue.add(', routeStart);
|
||||
const responseStart = serveSource.indexOf('res.status(202)', queueStart);
|
||||
expect(routeStart).toBeGreaterThanOrEqual(0);
|
||||
expect(queueStart).toBeGreaterThan(routeStart);
|
||||
expect(responseStart).toBeGreaterThan(queueStart);
|
||||
|
||||
const routeSource = serveSource.slice(queueStart, responseStart);
|
||||
const payload = routeSource.match(
|
||||
/queue\.add\(\s*'sync',\s*\{([\s\S]*?)\}\s*,\s*\{/,
|
||||
);
|
||||
expect(payload).not.toBeNull();
|
||||
expect(payload?.[1]).toMatch(/\bnoExtract:\s*false\b/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* #2849 — large-sync extract deferral queues an `extract --stale` follow-up.
|
||||
*
|
||||
* performSync's incremental path skips inline link/timeline extraction when
|
||||
* totalChanges > 100 (the #1794 large-sync deferral), leaving
|
||||
* links_extracted_at unstamped. Pre-fix, a standalone sync job (webhook push,
|
||||
* `gbrain sync trigger`) had NOTHING behind it to sweep those pages — the
|
||||
* autopilot cycle's extract phase only walks that cycle's changedSlugs — so a
|
||||
* large webhook push left extraction permanently stale until a manual
|
||||
* `gbrain extract --stale`.
|
||||
*
|
||||
* Pins:
|
||||
* (a) performSync surfaces `extractDeferred: true` on the >100 branch and
|
||||
* leaves the pages unstamped/unlinked.
|
||||
* (b) the `sync` job handler queues an `extract` job with
|
||||
* { stale: true, sourceId? } when extractDeferred is set.
|
||||
* (c) the `extract` handler's stale mode actually sweeps: links created +
|
||||
* watermark stamped (end-to-end recovery, no manual step).
|
||||
*
|
||||
* Marked .serial.test.ts — spawns git subprocesses + shares one PGLite engine.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionWorker } from '../src/core/minions/worker.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
import { registerBuiltinHandlers } from '../src/commands/jobs.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let worker: MinionWorker;
|
||||
let repoPath: string;
|
||||
|
||||
function git(cmd: string): void { execSync(cmd, { cwd: repoPath, stdio: 'pipe' }); }
|
||||
|
||||
describe('#2849 — large sync defers extract and queues a stale sweep', () => {
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
worker = new MinionWorker(engine, { queue: 'test' });
|
||||
await registerBuiltinHandlers(worker, engine, { quiet: true });
|
||||
|
||||
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-large-defer-'));
|
||||
git('git init');
|
||||
git('git config user.email "t@t.com"');
|
||||
git('git config user.name "T"');
|
||||
mkdirSync(join(repoPath, 'people'), { recursive: true });
|
||||
mkdirSync(join(repoPath, 'notes'), { recursive: true });
|
||||
writeFileSync(join(repoPath, 'people/alice.md'), [
|
||||
'---', 'type: person', 'title: Alice', '---', '', 'Alice is a founder.',
|
||||
].join('\n'));
|
||||
git('git add -A && git commit -m "initial"');
|
||||
|
||||
// Seed: full first sync imports the anchor page + sets last_commit.
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
|
||||
|
||||
// Second commit: 101 new pages → incremental totalChanges > 100.
|
||||
for (let i = 0; i < 101; i++) {
|
||||
writeFileSync(join(repoPath, `notes/n${i}.md`), [
|
||||
'---', 'type: note', `title: Note ${i}`, '---', '',
|
||||
`[Alice](people/alice) appears in note ${i}.`,
|
||||
].join('\n'));
|
||||
}
|
||||
git('git add -A && git commit -m "add 101 pages"');
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
test('sync handler defers inline extract and queues extract{stale} follow-up; stale sweep recovers', async () => {
|
||||
const syncHandler = (worker as unknown as { handlers: Map<string, (job: unknown) => Promise<unknown>> })
|
||||
.handlers.get('sync');
|
||||
expect(syncHandler).toBeDefined();
|
||||
|
||||
// Same payload shape the webhook submits (minus embed backfill noise).
|
||||
const result = await syncHandler!({
|
||||
data: { repoPath, noExtract: false, noPull: true, auto_embed_backfill: false },
|
||||
signal: { aborted: false },
|
||||
updateProgress: async () => {},
|
||||
}) as { status: string; extractDeferred?: boolean; extract_stale_job_id?: number | null };
|
||||
|
||||
expect(result.status).toBe('synced');
|
||||
// (a) inline extract was deferred, pages left stale.
|
||||
expect(result.extractDeferred).toBe(true);
|
||||
const staleBefore = await engine.countStalePagesForExtraction();
|
||||
expect(staleBefore).toBeGreaterThan(100);
|
||||
expect(await engine.getLinks('notes/n0')).toHaveLength(0);
|
||||
|
||||
// (b) a follow-up extract job with stale:true was queued.
|
||||
expect(result.extract_stale_job_id).toBeGreaterThan(0);
|
||||
const queue = new MinionQueue(engine);
|
||||
const extractJobs = await queue.getJobs({ name: 'extract', limit: 5 });
|
||||
expect(extractJobs.length).toBe(1);
|
||||
expect((extractJobs[0].data as { stale: boolean }).stale).toBe(true);
|
||||
|
||||
// (c) running the extract handler's stale mode recovers: links + stamps.
|
||||
const extractHandler = (worker as unknown as { handlers: Map<string, (job: unknown) => Promise<unknown>> })
|
||||
.handlers.get('extract');
|
||||
await extractHandler!({
|
||||
data: extractJobs[0].data,
|
||||
signal: { aborted: false },
|
||||
updateProgress: async () => {},
|
||||
});
|
||||
const links = await engine.getLinks('notes/n0');
|
||||
expect(links.some(l => l.to_slug === 'people/alice')).toBe(true);
|
||||
const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
|
||||
`SELECT links_extracted_at FROM pages WHERE slug = 'notes/n0'`,
|
||||
);
|
||||
expect(rows[0]?.links_extracted_at).not.toBeNull();
|
||||
}, 180_000);
|
||||
|
||||
test('sub-threshold sync does NOT set extractDeferred (no spurious follow-up)', async () => {
|
||||
// One more small commit → inline extract path, no deferral.
|
||||
writeFileSync(join(repoPath, 'notes/small.md'), [
|
||||
'---', 'type: note', 'title: Small', '---', '', 'No big deal.',
|
||||
].join('\n'));
|
||||
git('git add -A && git commit -m "one small page"');
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
expect(result.status).toBe('synced');
|
||||
expect(result.extractDeferred).toBeFalsy();
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -100,6 +100,10 @@ describe('runSyncTrigger', () => {
|
||||
const job = jobs[0];
|
||||
expect(job.priority).toBe(-10);
|
||||
expect((job.data as { sourceId: string }).sourceId).toBe('default');
|
||||
// #2849: opt in to inline extraction — the standalone sync handler
|
||||
// defaults noExtract to TRUE, which would leave triggered syncs
|
||||
// extraction-stale.
|
||||
expect((job.data as { noExtract: boolean }).noExtract).toBe(false);
|
||||
expect((job.data as { auto_embed_backfill: boolean }).auto_embed_backfill).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user