fix(webhook): extract links for incremental push syncs (#2849)

GitHub webhook + sync-trigger 'sync' jobs never opted into extraction:
the standalone sync handler defaults noExtract to TRUE, so webhook-
imported pages advanced the source bookmark with links_extracted_at
unstamped — and nothing behind a standalone job ever swept them (the
autopilot cycle's extract phase only walks that cycle's changedSlugs).

Fix, both halves of the issue:
- webhook + sync-trigger payloads pass noExtract:false so normal
  incremental pushes get sync's inline link/timeline extraction.
- >100-file deferral branch: performSync surfaces extractDeferred, and
  the sync job handler queues a source-scoped extract{stale:true}
  follow-up job (idempotent sweep; extract handler gains a stale mode
  backed by the same extractStaleFromDB core as `gbrain extract --stale`).

Tests: new sync-large-deferral-extract.serial.test.ts drives a real
101-file incremental sync end-to-end (deferral flag → queued follow-up →
stale sweep recovers links + watermark); webhook payload contract test +
sync-trigger noExtract assertion salvaged from #2850.

Fixes #2849
Takeover of #2850

Co-authored-by: patentsong <patentsong@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

llms-full.txt

@@ -1411,8 +1411,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
 | "get more out of gbrain", "is my brain set up right", "weekly brain checkup", "advise me on my brain", "gbrain advisor" | `skills/gbrain-advisor/SKILL.md` |
 | Save or load reports | `skills/reports/SKILL.md` |
 | "Create a skill", "improve this skill" | `skills/skill-creator/SKILL.md` |
-| "save this learning to the vault", "capture this skill in Obsidian", "record this workflow in my notes", "put this setup change in the vault" | `skills/skill-vault-capture-policy/SKILL.md` |
-| "Skillify this", "is this a skill?", "make this proper", "add tests and evals for this" | `skills/skillify/SKILL.md` |
+| "Skillify this", "is this a skill?", "make this proper" | `skills/skillify/SKILL.md` |
 | "Compress my resolver", "AGENTS.md too large", "RESOLVER.md too big", "functional area dispatcher", "shrink routing table" | `skills/functional-area-resolver/SKILL.md` |
 | "Is gbrain healthy?", morning health check, skillpack-check | `skills/skillpack-check/SKILL.md` |
 | "harvest this skill into gbrain", "publish this skill to gbrain", "lift this skill upstream", "share this skill with other gbrain clients", "promote my skill to gbrain" | `skills/skillpack-harvest/SKILL.md` |
@@ -1430,7 +1429,6 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
 | "Set up GBrain", first boot | `skills/setup/SKILL.md` |
 | "Now what?", "fill my brain", "cold start", "bootstrap", "import my data", "what should I import first" | `skills/cold-start/SKILL.md` |
 | "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
-| "Connect Obsidian to gbrain", "import my vault to gbrain", "sync vault and gbrain", "embed gbrain after vault update", "is gbrain synced with my vault" | `skills/obsidian-gbrain-safe-index/SKILL.md` |
 | Brain health check, maintenance run | `skills/maintain/SKILL.md` |
 | "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
 | "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |

openclaw.plugin.json

@@ -39,8 +39,8 @@
     "skills/briefing",
     "skills/citation-fixer",
     "skills/concept-synthesis",
-    "skills/cron-scheduler",
     "skills/cross-modal-review",
+    "skills/cron-scheduler",
     "skills/daily-task-manager",
     "skills/daily-task-prep",
     "skills/data-research",
@@ -54,11 +54,10 @@
     "skills/media-ingest",
     "skills/meeting-ingestion",
     "skills/minion-orchestrator",
-    "skills/obsidian-gbrain-safe-index",
     "skills/perplexity-research",
     "skills/query",
-    "skills/repo-architecture",
     "skills/reports",
+    "skills/repo-architecture",
     "skills/signal-detector",
     "skills/skill-creator",
     "skills/skillify",

skills/RESOLVER.md

@@ -60,8 +60,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
 | "get more out of gbrain", "is my brain set up right", "weekly brain checkup", "advise me on my brain", "gbrain advisor" | `skills/gbrain-advisor/SKILL.md` |
 | Save or load reports | `skills/reports/SKILL.md` |
 | "Create a skill", "improve this skill" | `skills/skill-creator/SKILL.md` |
-| "save this learning to the vault", "capture this skill in Obsidian", "record this workflow in my notes", "put this setup change in the vault" | `skills/skill-vault-capture-policy/SKILL.md` |
-| "Skillify this", "is this a skill?", "make this proper", "add tests and evals for this" | `skills/skillify/SKILL.md` |
+| "Skillify this", "is this a skill?", "make this proper" | `skills/skillify/SKILL.md` |
 | "Compress my resolver", "AGENTS.md too large", "RESOLVER.md too big", "functional area dispatcher", "shrink routing table" | `skills/functional-area-resolver/SKILL.md` |
 | "Is gbrain healthy?", morning health check, skillpack-check | `skills/skillpack-check/SKILL.md` |
 | "harvest this skill into gbrain", "publish this skill to gbrain", "lift this skill upstream", "share this skill with other gbrain clients", "promote my skill to gbrain" | `skills/skillpack-harvest/SKILL.md` |
@@ -79,7 +78,6 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
 | "Set up GBrain", first boot | `skills/setup/SKILL.md` |
 | "Now what?", "fill my brain", "cold start", "bootstrap", "import my data", "what should I import first" | `skills/cold-start/SKILL.md` |
 | "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
-| "Connect Obsidian to gbrain", "import my vault to gbrain", "sync vault and gbrain", "embed gbrain after vault update", "is gbrain synced with my vault" | `skills/obsidian-gbrain-safe-index/SKILL.md` |
 | Brain health check, maintenance run | `skills/maintain/SKILL.md` |
 | "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
 | "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |

skills/manifest.json

@@ -2,7 +2,7 @@
   "name": "gbrain",
   "version": "0.32.3.0",
   "conformance_version": "1.0.0",
-  "description": "Personal knowledge brain with hybrid RAG search — GStack mod for agent platforms",
+  "description": "Personal knowledge brain with hybrid RAG search \u2014 GStack mod for agent platforms",
   "skills": [
     {
       "name": "ingest",
@@ -34,11 +34,6 @@
       "path": "migrate/SKILL.md",
       "description": "Universal migration from Obsidian, Notion, Logseq, markdown, CSV, JSON, Roam"
     },
-    {
-      "name": "obsidian-gbrain-safe-index",
-      "path": "obsidian-gbrain-safe-index/SKILL.md",
-      "description": "Connect and sync an Obsidian-style Markdown vault with gbrain using a cost-controlled import-first workflow, explicit embedding gates, and durable skill/workflow capture."
-    },
     {
       "name": "setup",
       "path": "setup/SKILL.md",
@@ -268,11 +263,6 @@
       "name": "skill-optimizer",
       "path": "skill-optimizer/SKILL.md",
       "description": "Self-evolving skill optimization via gbrain skillopt — SkillOpt-paper-grounded text-space optimizer with validation gating (median-of-3 + epsilon=0.05), bundled-skill safety, bootstrap review sentinel, per-skill DB lock, and atomic versioned writes."
-    },
-    {
-      "name": "skill-vault-capture-policy",
-      "path": "skill-vault-capture-policy/SKILL.md",
-      "description": "Capture durable operational learnings, new agent skills, and environment/setup changes into the Obsidian vault instead of transient chat memory."
     }
   ],
   "dependencies": {

skills/obsidian-gbrain-safe-index/SKILL.md

@@ -1,137 +0,0 @@
----
-name: obsidian-gbrain-safe-index
-version: 1.0.0
-description: |
-  Connect, maintain, and sync an Obsidian-style Markdown vault with gbrain while preserving a cost-controlled workflow: the vault remains the source of truth, gbrain is the searchable/embedded index, durable skills/workflows are captured into the vault, and paid embedding runs only after explicit approval.
-triggers:
-  - "connect Obsidian to gbrain"
-  - "import my vault to gbrain"
-  - "sync vault and gbrain"
-  - "capture this skill in my vault"
-  - "embed gbrain after vault update"
-  - "is gbrain synced with my vault"
-tools:
-  - terminal
-  - read_file
-  - search_files
-  - write_file
-  - patch
-mutating: true
----
-
-# Obsidian → gbrain Safe Index and Capture
-
-## Contract
-
-This skill guarantees:
-
-- Treats the user's Obsidian-style Markdown vault as the source of truth before gbrain indexing.
-- Keeps gbrain in conservative mode unless the user explicitly approves a more expensive mode.
-- Imports vault changes with `--no-embed` first, then embeds only after explicit approval for the specific paid action.
-- Captures durable new skills, workflows, and environment learnings into the vault instead of leaving them only in chat memory.
-- Verifies every sync with concrete `gbrain stats`, search mode, and, when embeddings run, exact embedded chunk counts.
-
-## Phases
-
-1. **Resolve the vault path.**
-   - Prefer an existing environment variable such as `OBSIDIAN_VAULT_PATH` or `WIKI_PATH`.
-   - If no path is configured, search likely note directories and ask the user before writing.
-   - Verify the directory exists and contains markdown files or an `.obsidian` directory.
-
-2. **Read vault operating rules before writing.**
-   - If the vault has `SCHEMA.md`, `index.md`, `log.md`, `AGENTS.md`, or similar operating files, read them before ingest/query/major edit.
-   - Respect immutable source folders such as `raw/` when the vault declares them.
-   - Use the vault's native link convention, usually Obsidian `[[wikilinks]]`, for durable relationships.
-
-3. **MECE/capture decision.**
-   - If new knowledge belongs on an existing page, update that page.
-   - If it is a distinct recurring workflow or operational policy, create a small meta or concept page following the vault schema.
-   - Update the vault index/catalog for every new page when the vault maintains one.
-   - Append a log entry for meaningful vault updates when the vault maintains a log.
-
-4. **Safe gbrain import path.**
-   - Pre-check source directory; do not import a nonexistent path.
-   - Run `gbrain config set search.mode conservative` before/after risky reinit steps.
-   - Run `gbrain import "$OBSIDIAN_VAULT_PATH" --no-embed`.
-   - Run `gbrain extract links --source fs --dir "$OBSIDIAN_VAULT_PATH"` when wikilinks changed materially.
-
-5. **Paid embedding gate.**
-   - Do not run `gbrain embed --stale` unless the user explicitly asks or a prior instruction clearly approved this exact paid action.
-   - Before embedding, verify provider readiness with `gbrain providers test --model <provider:model>`.
-   - Confirm the configured embedding dimensions match the local schema.
-   - After embedding, verify `gbrain stats` and record exact `Pages`, `Chunks`, `Embedded`, and `Links` counts.
-
-6. **Final verification and vault echo.**
-   - Run `gbrain stats` and `gbrain search modes`.
-   - If vault files changed, re-import with `--no-embed`; if embedding was approved, embed stale chunks afterward.
-   - Report what changed, what was free/local, what used API billing, and what remains pending.
-
-## Output Format
-
-Use a compact status table:
-
-| Item | Status |
-|---|---|
-| Vault path | `/path` |
-| Vault updated | yes/no + files |
-| gbrain mode | conservative/balanced/tokenmax |
-| Import | `--no-embed` completed / skipped / failed |
-| Pages/chunks | exact counts from `gbrain stats` |
-| Embeddings | exact count; note whether this run used API billing |
-| Links | exact count |
-| Background jobs | none / list exact jobs |
-
-Then include:
-
-- **Safe next step:** free/local action.
-- **Paid next step:** embedding/LLM action, if any, with explicit approval requirement.
-
-## Anti-Patterns
-
-- Creating a duplicate skill/page when an existing Obsidian, gbrain, or vault-ingest skill already covers the workflow.
-- Running `gbrain embed --stale`, `gbrain dream`, `gbrain autopilot --install`, `gbrain onboard --auto`, or `tokenmax` without explicit cost approval.
-- Importing a nonexistent or wrong directory and treating a zero-page import as success.
-- Forgetting to update the vault index/catalog and log after creating or materially updating vault pages.
-- Recording API keys, tokens, or raw secrets in the vault or final response.
-
-## Tools Used
-
-- `read_file` — read vault schema/index/log and target notes.
-- `search_files` — find existing vault pages and avoid duplicates.
-- `write_file` / `patch` — create or update vault pages.
-- `terminal` — run `gbrain`, `git`, and environment checks with secret values redacted.
-
-## Safe Commands
-
-```bash
-export PATH="$HOME/.bun/bin:$PATH"
-gbrain config set search.mode conservative
-gbrain import "$OBSIDIAN_VAULT_PATH" --no-embed
-gbrain extract links --source fs --dir "$OBSIDIAN_VAULT_PATH"
-gbrain stats
-gbrain search modes
-```
-
-## Paid / Approval-Gated Commands
-
-```bash
-gbrain providers test --model <provider:model>
-gbrain embed --stale
-gbrain dream
-gbrain autopilot --install
-gbrain onboard --auto --max-usd 5
-gbrain config set search.mode tokenmax
-```
-
-## Verification Checklist
-
-- [ ] Vault path exists and is the intended source.
-- [ ] Vault operating files were read before edits when present.
-- [ ] Existing pages/skills were searched to avoid duplicates.
-- [ ] New/updated vault pages follow the vault schema and link convention.
-- [ ] Vault index/catalog updated for new pages when present.
-- [ ] Vault log appended for meaningful actions when present.
-- [ ] `gbrain import ... --no-embed` completed.
-- [ ] `gbrain stats` recorded pages/chunks/embeddings/links.
-- [ ] `gbrain search modes` confirms conservative mode unless a different mode was explicitly approved.
-- [ ] No paid/background commands ran without approval.

skills/obsidian-gbrain-safe-index/routing-eval.jsonl

@@ -1,11 +0,0 @@
-// Routing eval fixtures for skills/obsidian-gbrain-safe-index.
-// Positive cases: intents embed a trigger phrase in natural surrounding context
-// (never verbatim-identical to a trigger — the fixture linter rejects tautologies).
-{"intent": "help me connect Obsidian to gbrain for my notes", "expected_skill": "obsidian-gbrain-safe-index"}
-{"intent": "import my vault to gbrain but skip embeddings for now", "expected_skill": "obsidian-gbrain-safe-index"}
-{"intent": "please sync vault and gbrain after I edit notes", "expected_skill": "obsidian-gbrain-safe-index"}
-{"intent": "run embed gbrain after vault update tonight", "expected_skill": "obsidian-gbrain-safe-index"}
-{"intent": "hey is gbrain synced with my vault right now", "expected_skill": "obsidian-gbrain-safe-index"}
-// Negative cases: related but owned by other skills. Assert NO route to this skill.
-{"intent": "migrate my notes from Notion to gbrain", "expected_skill": null}
-{"intent": "what is on my calendar tomorrow", "expected_skill": null}

skills/skill-vault-capture-policy/SKILL.md

@@ -1,100 +0,0 @@
----
-name: skill-vault-capture-policy
-version: 1.0.0
-description: |
-  Use when the user wants a durable operational learning, new agent skill, or
-  important environment/setup change to be captured into the Obsidian vault
-  instead of left only in transient chat memory. Covers the capture rule,
-  preferred page patterns, and index/log update obligations.
-triggers:
-  - "save this learning to the vault"
-  - "capture this skill in Obsidian"
-  - "record this workflow in my notes"
-  - "put this setup change in the vault"
-  - "should we add this to the knowledge base"
-tools:
-  - read_file
-  - search_files
-  - write_file
-  - patch
-mutating: true
----
-
-# Skill Vault Capture Policy
-
-## Contract
-
-This skill guarantees:
-
-- Durable operational learnings, newly adopted skills, and important environment/setup changes are captured into the Obsidian vault rather than left only in chat memory.
-- An existing page is updated when the knowledge clearly belongs there; a new page is created only when the topic is distinct and likely to recur.
-- Every new vault page is added to `index.md`.
-- Every meaningful create/update appends a dated entry to `log.md`.
-- Small linked pages are preferred over one giant running note.
-
-## Phases
-
-1. **Classify the learning.**
-   - New gbrain operating rule, cost control, or embedding/provider change.
-   - New agent-fork / harness / coding-tool integration fact.
-   - New Obsidian vault workflow or structure decision.
-   - New recurring agent skill that changes how the agent should operate here.
-
-2. **Avoid duplicates.**
-   - Search the vault for an existing page that already owns the topic.
-   - If found, update it with a new section or dated note rather than creating a near-duplicate.
-
-3. **Create when distinct.**
-   - Place new pages under the vault schema: `_meta/` for operating notes, `concepts/` for workflows, `entities/` for tools/people.
-   - Use YAML frontmatter and at least two `[[wikilinks]]` unless it is a short seed page.
-
-4. **Update navigation.**
-   - Add the page to `index.md` under the correct type heading.
-   - Append a `## [YYYY-MM-DD] create|update | subject` entry to `log.md`.
-
-5. **Report the capture.**
-   - State which files changed and whether `index.md` / `log.md` were updated.
-
-## Output Format
-
-Use a short status block:
-
-| Item | Status |
-|---|---|
-| Learning classified | type |
-| Page created/updated | path |
-| index.md updated | yes/no |
-| log.md updated | yes/no |
-
-## Anti-Patterns
-
-- Leaving durable learnings only in chat memory.
-- Creating a near-duplicate page instead of updating the existing one.
-- Forgetting to update `index.md` and `log.md`.
-- Writing one giant running note instead of small linked pages.
-- Recording secrets, API keys, or raw credentials in the vault.
-
-## Tools Used
-
-- `read_file` — read `SCHEMA.md`, `index.md`, `log.md`, and target pages.
-- `search_files` — find existing pages to avoid duplicates.
-- `write_file` / `patch` — create or update vault pages and navigation.
-
-## Safe Commands
-
-```bash
-# inspect vault navigation before writing
-read SCHEMA.md index.md log.md
-# create or update a page, then refresh catalog/log
-# index.md: add [[page-slug]] under the matching type heading
-# log.md: append ## [YYYY-MM-DD] create|update | subject
-```
-
-## Verification Checklist
-
-- [ ] Vault schema/read files were checked before writing.
-- [ ] Existing pages were searched to avoid duplicates.
-- [ ] New/updated page has frontmatter and wikilinks.
-- [ ] `index.md` updated for new pages.
-- [ ] `log.md` appended for meaningful actions.
-- [ ] No secrets or raw credentials were written.

skills/skill-vault-capture-policy/routing-eval.jsonl

@@ -1,10 +0,0 @@
-// Routing eval fixtures for skills/skill-vault-capture-policy.
-// Positive cases: intents embed a trigger phrase in natural surrounding context
-// (never verbatim-identical to a trigger — the fixture linter rejects tautologies).
-{"intent": "please save this learning to the vault so we keep it", "expected_skill": "skill-vault-capture-policy", "ambiguous_with": ["idea-ingest"]}
-{"intent": "we should capture this skill in Obsidian for reuse", "expected_skill": "skill-vault-capture-policy", "ambiguous_with": ["capture"]}
-{"intent": "can you record this workflow in my notes for next time", "expected_skill": "skill-vault-capture-policy"}
-{"intent": "put this setup change in the vault before we forget", "expected_skill": "skill-vault-capture-policy"}
-// Negative cases: related but owned by other skills or out of scope.
-{"intent": "connect my Obsidian vault to gbrain", "expected_skill": null}
-{"intent": "what is on my calendar tomorrow", "expected_skill": null}

src/commands/extract.ts

@@ -1651,7 +1651,7 @@ async function extractTimelineFromDB(
  * make re-extraction idempotent). EVERY processed page is stamped, including
  * zero-link pages — they WERE processed.
  */
-async function extractStaleFromDB(
+export async function extractStaleFromDB(
   engine: BrainEngine,
   opts: {
     dryRun: boolean;

src/commands/jobs.ts

@@ -1479,7 +1479,31 @@ export async function registerBuiltinHandlers(
       embedSkipReason = 'auto_embed_disabled';
     }
 
-    return { ...result, embed_job_id: embedJobId, embed_skip_reason: embedSkipReason };
+    // #2849: large-sync extract deferral follow-up. performSync skips inline
+    // link/timeline extraction when totalChanges > 100, leaving
+    // links_extracted_at unstamped. A standalone sync job (webhook push,
+    // sync trigger) has no autopilot extract phase behind it, so the pages
+    // would stay extraction-stale until a manual `gbrain extract --stale`.
+    // Queue a source-scoped stale sweep instead. Best-effort + idempotent:
+    // a duplicate sweep finds 0 stale pages and no-ops.
+    let extractJobId: number | null = null;
+    if (result.extractDeferred) {
+      try {
+        const { MinionQueue } = await import('../core/minions/queue.ts');
+        const queue = new MinionQueue(engine);
+        const followUp = await queue.add(
+          'extract',
+          { stale: true, ...(sourceId ? { sourceId } : {}) },
+          {
+            idempotency_key: `sync-extract-stale:${sourceId ?? 'default'}:${Math.floor(Date.now() / 30_000)}`,
+            maxWaiting: 1,
+          },
+        );
+        extractJobId = followUp.id;
+      } catch { /* best-effort: extract --stale sweeps it later */ }
+    }
+
+    return { ...result, embed_job_id: embedJobId, embed_skip_reason: embedSkipReason, extract_stale_job_id: extractJobId };
   });
 
   registerBuiltinJob(worker, engine, 'embed', async (job) => {
@@ -1652,6 +1676,20 @@ export async function registerBuiltinHandlers(
   });
 
   worker.register('extract', async (job) => {
+    // #2849: stale-sweep mode — the sync handler's large-sync deferral
+    // follow-up. DB-source (reads page content from the DB, so it runs on
+    // checkout-less brains), source-scopable, idempotent. Same core as
+    // `gbrain extract --stale`.
+    if (job.data.stale === true) {
+      const { extractStaleFromDB } = await import('./extract.ts');
+      return await extractStaleFromDB(engine, {
+        dryRun: !!job.data.dryRun,
+        jsonMode: false,
+        includeFrontmatter: false,
+        sourceIdFilter: typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined,
+        catchUp: false,
+      });
+    }
     const { runExtractCore } = await import('./extract.ts');
     const mode = (typeof job.data.mode === 'string' && ['links', 'timeline', 'all'].includes(job.data.mode))
       ? (job.data.mode as 'links' | 'timeline' | 'all')

src/commands/serve-http.ts

@@ -2146,8 +2146,13 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
   //     Other event types (ping, pull_request, etc.) return 202 'ignored'
   //     so GitHub doesn't retry.
   // D15.5: HMAC compare uses the shared safeHexEqual helper.
-  // D18: submits 'sync' job with auto_embed_backfill=true and priority -10
-  //     (above autopilot's 0).
+  // D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
+  //     priority -10 (above autopilot's 0). noExtract:false opts normal
+  //     incremental pushes into sync's inline link/timeline extraction (#2849
+  //     — the standalone sync handler defaults noExtract to TRUE, which left
+  //     webhook-imported pages permanently stale). Large (>100 file) pushes
+  //     defer inline extract; the sync handler queues an extract --stale
+  //     follow-up job for that branch.
   // ---------------------------------------------------------------------------
   const githubWebhookLimiter = rateLimit({
     windowMs: 60_000,
@@ -2267,6 +2272,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
           'sync',
           {
             sourceId: source.id,
+            noExtract: false,
             auto_embed_backfill: true,
             embed_reason: 'webhook',
           },

src/commands/sync.ts

@@ -222,6 +222,14 @@ export interface SyncResult {
    * everything," the exact misdiagnosis in the #1794 recurrence report.
    */
   bankedFiles?: number;
+  /**
+   * #2849: true when extraction was REQUESTED (noExtract false) but this sync
+   * skipped inline link/timeline extraction because totalChanges > 100 (the
+   * #1794 large-sync deferral). links_extracted_at stays unstamped for the
+   * imported pages. The standalone `sync` job handler queues a source-scoped
+   * `extract --stale` follow-up when set; CLI runs print the manual hint.
+   */
+  extractDeferred?: boolean;
 }
 
 /**
@@ -1379,6 +1387,10 @@ See also:
     {
       sourceId: sourceIdArg,
       repoPath: source.local_path,
+      // #2849: opt in to inline extraction — the standalone sync handler
+      // defaults noExtract to TRUE (dedupe for doctor's [sync, extract]
+      // remediation plan), which would leave triggered syncs extraction-stale.
+      noExtract: false,
       auto_embed_backfill: true,
       embed_reason: 'sync_trigger',
     },
@@ -3287,11 +3299,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
   // the stale sweep scans the whole source, so banked-across-runs pages are
   // covered regardless.
   const extractOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
+  let extractDeferred = false;
   if (!opts.noExtract && totalChanges > 100 && pagesAffected.length > 0) {
+    // #2849: surface the deferral to callers. A standalone sync job (webhook
+    // push, sync trigger) has no autopilot extract phase behind it, so the
+    // job handler queues an `extract --stale` follow-up off this flag.
+    extractDeferred = true;
     slog(
       `  Large sync: deferring link/timeline extraction. ` +
       `Run 'gbrain extract --stale${opts.sourceId ? ` --source-id ${opts.sourceId}` : ''}' ` +
-      `(or let the autopilot cycle's extract phase sweep it).`,
+      `(sync jobs queue this follow-up automatically).`,
     );
   }
   if (!opts.noExtract && totalChanges <= 100 && pagesAffected.length > 0) {
@@ -3400,6 +3417,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
     chunksCreated,
     embedded,
     pagesAffected,
+    extractDeferred,
   };
 }
 

test/e2e/obsidian-gbrain-safe-index.test.ts

@@ -1,61 +0,0 @@
-/**
- * E2E smoke for skills/obsidian-gbrain-safe-index.
- *
- * Verifies the from-trigger-to-side-effect path that skillify requires:
- * a real user trigger phrase routes to the skill, the resolver/check
- * pipeline treats it as reachable, and the skill file exposes the
- * gbrain commands the workflow actually runs.
- *
- * This stays local-only (no paid embedding, no external API): it asserts
- * the documented safe/import path is present and parseable, not that it
- * mutates a live brain.
- */
-
-import { describe, expect, it } from 'bun:test';
-import { existsSync, readFileSync } from 'fs';
-import { join } from 'path';
-
-const SKILLS = join(import.meta.dir, '..', '..', 'skills');
-const SKILL_MD = join(SKILLS, 'obsidian-gbrain-safe-index', 'SKILL.md');
-const RESOLVER = join(SKILLS, 'RESOLVER.md');
-
-const TRIGGER_PHRASES = [
-  'connect my Obsidian vault to gbrain',
-  'import my vault to gbrain',
-  'sync vault and gbrain',
-  'capture this skill in my vault',
-  'embed gbrain after vault update',
-  'is gbrain synced with my vault',
-];
-
-describe('obsidian-gbrain-safe-index E2E', () => {
-  it('resolver maps real trigger phrasings to the skill', () => {
-    const resolver = readFileSync(RESOLVER, 'utf-8');
-    expect(resolver).toContain('obsidian-gbrain-safe-index/SKILL.md');
-    // Each representative phrase shares a token substring with a resolver row.
-    const rows = resolver
-      .split('\n')
-      .filter((l) => l.includes('obsidian-gbrain-safe-index/SKILL.md'))
-      .join('\n');
-    for (const phrase of TRIGGER_PHRASES) {
-      const hit = phrase
-        .toLowerCase()
-        .split(/\s+/)
-        .some((tok) => tok.length > 3 && rows.toLowerCase().includes(tok));
-      expect(hit, `no resolver token for: ${phrase}`).toBe(true);
-    }
-  });
-
-  it('skill documents the gbrain import-first safe path', () => {
-    const body = readFileSync(SKILL_MD, 'utf-8');
-    expect(body).toContain('gbrain import');
-    expect(body).toContain('--no-embed');
-    expect(body).toContain('gbrain config set search.mode conservative');
-    // Paid gate must be explicit, not a silent default.
-    expect(body).toContain('gbrain embed --stale');
-  });
-
-  it('skill is reachable from the skill tree', () => {
-    expect(existsSync(SKILL_MD)).toBe(true);
-  });
-});

test/e2e/skill-vault-capture-policy.test.ts

@@ -1,52 +0,0 @@
-/**
- * E2E smoke for skills/skill-vault-capture-policy.
- *
- * Verifies the from-trigger-to-side-effect path: a real capture request
- * routes to the skill, and the skill documents the vault navigation
- * obligations (index.md / log.md) that make a capture durable.
- *
- * Local-only: it asserts documented behavior, not live vault writes.
- */
-
-import { describe, expect, it } from 'bun:test';
-import { existsSync, readFileSync } from 'fs';
-import { join } from 'path';
-
-const SKILLS = join(import.meta.dir, '..', '..', 'skills');
-const SKILL_MD = join(SKILLS, 'skill-vault-capture-policy', 'SKILL.md');
-const RESOLVER = join(SKILLS, 'RESOLVER.md');
-
-const TRIGGER_PHRASES = [
-  'save this learning to the vault',
-  'capture this skill in Obsidian',
-  'record this workflow in my notes',
-  'put this setup change in the knowledge base',
-];
-
-describe('skill-vault-capture-policy E2E', () => {
-  it('resolver maps real capture phrasings to the skill', () => {
-    const resolver = readFileSync(RESOLVER, 'utf-8');
-    expect(resolver).toContain('skill-vault-capture-policy/SKILL.md');
-    const rows = resolver
-      .split('\n')
-      .filter((l) => l.includes('skill-vault-capture-policy/SKILL.md'))
-      .join('\n');
-    for (const phrase of TRIGGER_PHRASES) {
-      const hit = phrase
-        .toLowerCase()
-        .split(/\s+/)
-        .some((tok) => tok.length > 3 && rows.toLowerCase().includes(tok));
-      expect(hit, `no resolver token for: ${phrase}`).toBe(true);
-    }
-  });
-
-  it('skill documents index.md and log.md update obligations', () => {
-    const body = readFileSync(SKILL_MD, 'utf-8');
-    expect(body).toContain('index.md');
-    expect(body).toContain('log.md');
-  });
-
-  it('skill is reachable from the skill tree', () => {
-    expect(existsSync(SKILL_MD)).toBe(true);
-  });
-});

test/obsidian-gbrain-safe-index.test.ts

@@ -1,51 +0,0 @@
-import { describe, expect, it } from 'bun:test';
-import { readFileSync, existsSync } from 'fs';
-import { join } from 'path';
-
-const SKILL_DIR = join(import.meta.dir, '..', 'skills', 'obsidian-gbrain-safe-index');
-const SKILL_MD = join(SKILL_DIR, 'SKILL.md');
-const RESOLVER = join(import.meta.dir, '..', 'skills', 'RESOLVER.md');
-
-function parseFrontmatter(raw: string): Record<string, unknown> {
-  const m = raw.match(/^---\n([\s\S]*?)\n---/);
-  if (!m) throw new Error('no frontmatter');
-  const out: Record<string, unknown> = {};
-  for (const line of m[1].split('\n')) {
-    const mm = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
-    if (mm) out[mm[1]] = mm[2].trim();
-  }
-  return out;
-}
-
-describe('obsidian-gbrain-safe-index skill', () => {
-  it('has a SKILL.md with required frontmatter', () => {
-    expect(existsSync(SKILL_MD)).toBe(true);
-    const fm = parseFrontmatter(readFileSync(SKILL_MD, 'utf-8'));
-    expect(fm['name']).toBe('obsidian-gbrain-safe-index');
-    expect(fm['description']).toBeTruthy();
-  });
-
-  it('has the required conformance sections', () => {
-    const body = readFileSync(SKILL_MD, 'utf-8');
-    for (const section of ['## Contract', '## Phases', '## Output Format', '## Anti-Patterns']) {
-      expect(body.includes(section), `missing ${section}`).toBe(true);
-    }
-  });
-
-  it('is registered in RESOLVER.md', () => {
-    expect(existsSync(RESOLVER)).toBe(true);
-    const resolver = readFileSync(RESOLVER, 'utf-8');
-    expect(resolver.includes('obsidian-gbrain-safe-index/SKILL.md')).toBe(true);
-  });
-
-  it('has routing-eval fixtures that exercise real trigger phrasings', () => {
-    const evalPath = join(SKILL_DIR, 'routing-eval.jsonl');
-    expect(existsSync(evalPath)).toBe(true);
-    const lines = readFileSync(evalPath, 'utf-8')
-      .split('\n')
-      .filter((l) => l.trim() && !l.trim().startsWith('//'))
-      .map((l) => JSON.parse(l));
-    const positives = lines.filter((l) => l.expected_skill === 'obsidian-gbrain-safe-index');
-    expect(positives.length).toBeGreaterThanOrEqual(5);
-  });
-});

test/skill-vault-capture-policy.test.ts

@@ -1,64 +0,0 @@
-import { describe, expect, it } from 'bun:test';
-import { readFileSync, existsSync } from 'fs';
-import { join } from 'path';
-import { DEFAULT_PRIVATE_PATTERNS } from '../src/core/skillpack/harvest-lint.ts';
-
-const SKILL_DIR = join(import.meta.dir, '..', 'skills', 'skill-vault-capture-policy');
-const SKILL_MD = join(SKILL_DIR, 'SKILL.md');
-const RESOLVER = join(import.meta.dir, '..', 'skills', 'RESOLVER.md');
-
-function parseFrontmatter(raw: string): Record<string, unknown> {
-  const m = raw.match(/^---\n([\s\S]*?)\n---/);
-  if (!m) throw new Error('no frontmatter');
-  const out: Record<string, unknown> = {};
-  for (const line of m[1].split('\n')) {
-    const mm = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
-    if (mm) out[mm[1]] = mm[2].trim();
-  }
-  return out;
-}
-
-describe('skill-vault-capture-policy skill', () => {
-  it('has a SKILL.md with required frontmatter', () => {
-    expect(existsSync(SKILL_MD)).toBe(true);
-    const fm = parseFrontmatter(readFileSync(SKILL_MD, 'utf-8'));
-    expect(fm['name']).toBe('skill-vault-capture-policy');
-    expect(fm['description']).toBeTruthy();
-  });
-
-  it('has the required conformance sections', () => {
-    const body = readFileSync(SKILL_MD, 'utf-8');
-    for (const section of ['## Contract', '## Phases', '## Output Format', '## Anti-Patterns']) {
-      expect(body.includes(section), `missing ${section}`).toBe(true);
-    }
-  });
-
-  it('is registered in RESOLVER.md', () => {
-    expect(existsSync(RESOLVER)).toBe(true);
-    expect(readFileSync(RESOLVER, 'utf-8').includes('skill-vault-capture-policy/SKILL.md')).toBe(true);
-  });
-
-  it('contains no private user or agent-fork names (privacy rule)', () => {
-    for (const file of [SKILL_MD, join(SKILL_DIR, 'routing-eval.jsonl')]) {
-      const body = readFileSync(file, 'utf-8');
-      // DEFAULT_PRIVATE_PATTERNS[0] is the banned fork-name pattern; sourced
-      // from harvest-lint so this file never contains the literal itself
-      // (scripts/check-privacy.sh would reject it).
-      const forkName = new RegExp(DEFAULT_PRIVATE_PATTERNS[0], 'i');
-      for (const name of [/\bAdam\b/, /\bHermes\b/, /\bHerdr\b/, /\bArk\b/, forkName]) {
-        expect(name.test(body), `private name ${name} in ${file}`).toBe(false);
-      }
-    }
-  });
-
-  it('has routing-eval fixtures', () => {
-    const evalPath = join(SKILL_DIR, 'routing-eval.jsonl');
-    expect(existsSync(evalPath)).toBe(true);
-    const positives = readFileSync(evalPath, 'utf-8')
-      .split('\n')
-      .filter((l) => l.trim() && !l.trim().startsWith('//'))
-      .map((l) => JSON.parse(l))
-      .filter((l) => l.expected_skill === 'skill-vault-capture-policy');
-    expect(positives.length).toBeGreaterThanOrEqual(4);
-  });
-});

test/sources-webhook.test.ts

@@ -16,6 +16,7 @@
  */
 import { describe, test, expect } from 'bun:test';
 import { createHmac } from 'node:crypto';
+import { readFileSync } from 'node:fs';
 import { safeHexEqual } from '../src/core/timing-safe.ts';
 
 const GITHUB_SECRET = 'super-secret-webhook-key';
@@ -123,3 +124,25 @@ describe('Branch ref construction (D5)', () => {
     expect(pushedRef === `refs/heads/${trackedBranch}`).toBe(false);
   });
 });
+
+describe('Webhook sync job extraction contract (#2849)', () => {
+  test('opts into extraction before the pushed commit is consumed', () => {
+    const serveSource = readFileSync(
+      new URL('../src/commands/serve-http.ts', import.meta.url),
+      'utf8',
+    );
+    const routeStart = serveSource.indexOf("'/webhooks/github'");
+    const queueStart = serveSource.indexOf('const job = await queue.add(', routeStart);
+    const responseStart = serveSource.indexOf('res.status(202)', queueStart);
+    expect(routeStart).toBeGreaterThanOrEqual(0);
+    expect(queueStart).toBeGreaterThan(routeStart);
+    expect(responseStart).toBeGreaterThan(queueStart);
+
+    const routeSource = serveSource.slice(queueStart, responseStart);
+    const payload = routeSource.match(
+      /queue\.add\(\s*'sync',\s*\{([\s\S]*?)\}\s*,\s*\{/,
+    );
+    expect(payload).not.toBeNull();
+    expect(payload?.[1]).toMatch(/\bnoExtract:\s*false\b/);
+  });
+});

test/sync-large-deferral-extract.serial.test.ts

@@ -0,0 +1,130 @@
+/**
+ * #2849 — large-sync extract deferral queues an `extract --stale` follow-up.
+ *
+ * performSync's incremental path skips inline link/timeline extraction when
+ * totalChanges > 100 (the #1794 large-sync deferral), leaving
+ * links_extracted_at unstamped. Pre-fix, a standalone sync job (webhook push,
+ * `gbrain sync trigger`) had NOTHING behind it to sweep those pages — the
+ * autopilot cycle's extract phase only walks that cycle's changedSlugs — so a
+ * large webhook push left extraction permanently stale until a manual
+ * `gbrain extract --stale`.
+ *
+ * Pins:
+ *   (a) performSync surfaces `extractDeferred: true` on the >100 branch and
+ *       leaves the pages unstamped/unlinked.
+ *   (b) the `sync` job handler queues an `extract` job with
+ *       { stale: true, sourceId? } when extractDeferred is set.
+ *   (c) the `extract` handler's stale mode actually sweeps: links created +
+ *       watermark stamped (end-to-end recovery, no manual step).
+ *
+ * Marked .serial.test.ts — spawns git subprocesses + shares one PGLite engine.
+ */
+
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
+import { execSync } from 'child_process';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { MinionWorker } from '../src/core/minions/worker.ts';
+import { MinionQueue } from '../src/core/minions/queue.ts';
+import { registerBuiltinHandlers } from '../src/commands/jobs.ts';
+
+let engine: PGLiteEngine;
+let worker: MinionWorker;
+let repoPath: string;
+
+function git(cmd: string): void { execSync(cmd, { cwd: repoPath, stdio: 'pipe' }); }
+
+describe('#2849 — large sync defers extract and queues a stale sweep', () => {
+  beforeAll(async () => {
+    engine = new PGLiteEngine();
+    await engine.connect({});
+    await engine.initSchema();
+    worker = new MinionWorker(engine, { queue: 'test' });
+    await registerBuiltinHandlers(worker, engine, { quiet: true });
+
+    repoPath = mkdtempSync(join(tmpdir(), 'gbrain-large-defer-'));
+    git('git init');
+    git('git config user.email "t@t.com"');
+    git('git config user.name "T"');
+    mkdirSync(join(repoPath, 'people'), { recursive: true });
+    mkdirSync(join(repoPath, 'notes'), { recursive: true });
+    writeFileSync(join(repoPath, 'people/alice.md'), [
+      '---', 'type: person', 'title: Alice', '---', '', 'Alice is a founder.',
+    ].join('\n'));
+    git('git add -A && git commit -m "initial"');
+
+    // Seed: full first sync imports the anchor page + sets last_commit.
+    const { performSync } = await import('../src/commands/sync.ts');
+    await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
+
+    // Second commit: 101 new pages → incremental totalChanges > 100.
+    for (let i = 0; i < 101; i++) {
+      writeFileSync(join(repoPath, `notes/n${i}.md`), [
+        '---', 'type: note', `title: Note ${i}`, '---', '',
+        `[Alice](people/alice) appears in note ${i}.`,
+      ].join('\n'));
+    }
+    git('git add -A && git commit -m "add 101 pages"');
+  }, 120_000);
+
+  afterAll(async () => {
+    if (repoPath) rmSync(repoPath, { recursive: true, force: true });
+    if (engine) await engine.disconnect();
+  }, 60_000);
+
+  test('sync handler defers inline extract and queues extract{stale} follow-up; stale sweep recovers', async () => {
+    const syncHandler = (worker as unknown as { handlers: Map<string, (job: unknown) => Promise<unknown>> })
+      .handlers.get('sync');
+    expect(syncHandler).toBeDefined();
+
+    // Same payload shape the webhook submits (minus embed backfill noise).
+    const result = await syncHandler!({
+      data: { repoPath, noExtract: false, noPull: true, auto_embed_backfill: false },
+      signal: { aborted: false },
+      updateProgress: async () => {},
+    }) as { status: string; extractDeferred?: boolean; extract_stale_job_id?: number | null };
+
+    expect(result.status).toBe('synced');
+    // (a) inline extract was deferred, pages left stale.
+    expect(result.extractDeferred).toBe(true);
+    const staleBefore = await engine.countStalePagesForExtraction();
+    expect(staleBefore).toBeGreaterThan(100);
+    expect(await engine.getLinks('notes/n0')).toHaveLength(0);
+
+    // (b) a follow-up extract job with stale:true was queued.
+    expect(result.extract_stale_job_id).toBeGreaterThan(0);
+    const queue = new MinionQueue(engine);
+    const extractJobs = await queue.getJobs({ name: 'extract', limit: 5 });
+    expect(extractJobs.length).toBe(1);
+    expect((extractJobs[0].data as { stale: boolean }).stale).toBe(true);
+
+    // (c) running the extract handler's stale mode recovers: links + stamps.
+    const extractHandler = (worker as unknown as { handlers: Map<string, (job: unknown) => Promise<unknown>> })
+      .handlers.get('extract');
+    await extractHandler!({
+      data: extractJobs[0].data,
+      signal: { aborted: false },
+      updateProgress: async () => {},
+    });
+    const links = await engine.getLinks('notes/n0');
+    expect(links.some(l => l.to_slug === 'people/alice')).toBe(true);
+    const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
+      `SELECT links_extracted_at FROM pages WHERE slug = 'notes/n0'`,
+    );
+    expect(rows[0]?.links_extracted_at).not.toBeNull();
+  }, 180_000);
+
+  test('sub-threshold sync does NOT set extractDeferred (no spurious follow-up)', async () => {
+    // One more small commit → inline extract path, no deferral.
+    writeFileSync(join(repoPath, 'notes/small.md'), [
+      '---', 'type: note', 'title: Small', '---', '', 'No big deal.',
+    ].join('\n'));
+    git('git add -A && git commit -m "one small page"');
+    const { performSync } = await import('../src/commands/sync.ts');
+    const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
+    expect(result.status).toBe('synced');
+    expect(result.extractDeferred).toBeFalsy();
+  }, 60_000);
+});

test/sync-trigger-cli.test.ts

@@ -100,6 +100,10 @@ describe('runSyncTrigger', () => {
     const job = jobs[0];
     expect(job.priority).toBe(-10);
     expect((job.data as { sourceId: string }).sourceId).toBe('default');
+    // #2849: opt in to inline extraction — the standalone sync handler
+    // defaults noExtract to TRUE, which would leave triggered syncs
+    // extraction-stale.
+    expect((job.data as { noExtract: boolean }).noExtract).toBe(false);
     expect((job.data as { auto_embed_backfill: boolean }).auto_embed_backfill).toBe(true);
   });