Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 2ba63ddec5 fix(cycle): honor DB-plane config for incremental_extract_include_frontmatter
The gate read loadConfig() (file/env plane) only, but the documented enable
command — gbrain config set autopilot.incremental_extract_include_frontmatter
true — writes the DB plane via engine.setConfig, so the feature could never be
turned on the documented way (silent no-op, #2120 class). Now the file plane
wins when the key is present there; otherwise the DB plane is consulted,
matching the autopilot.auto_drain.* read pattern.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 12:16:01 -07:00
214449b190 fix(links): resolve [[wikilink]] + slug-path frontmatter values; keep frontmatter links fresh on the incremental cycle
Takeover/rebase of two community PRs:

PR #1983 — frontmatter link fields never resolved Obsidian-style values:
- makeResolver step 1's strict slug regex rejected digit-leading folders
  (90-people/nicolai) and nested paths (a/b/c); broadened to any slug-shaped
  value with an EXACT getPage match only (no fuzzy, no false positives).
- extractFrontmatterLinks resolved "[[dir/slug]]" verbatim; new anchored
  unwrapWikilink() strips wholly-wrapped [[...]] (and |alias/#heading/^block)
  before resolution. Bare values pass through unchanged.
- Same broadened slug-shape applied to the fs-path synthetic resolver in
  extractLinksFromFile (exact Set membership guards it), so the fs
  frontmatter path resolves PARA-numbered slugs too.

PR #2434 — the cycle's incremental extract (extractForSlugs) extracted body
links only, so externally-edited YAML (sources:/related:) edges drifted
stale. Adds an includeFrontmatter opt (threaded as a param after sourceId,
which master added in #1747/#1503 after the PR was cut), gated by the new
config key autopilot.incremental_extract_include_frontmatter (default off,
preserves body-only behavior).

Tests: unwrapWikilink unit coverage, broadened-resolver + end-to-end
frontmatter cases in test/link-extraction.test.ts; fs-resolver digit-leading
case in test/extract.test.ts; incremental gate off/on cases in
test/extract-incremental.test.ts.

Co-authored-by: spiky02plateau <spiky02plateau@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:23:10 -07:00
19 changed files with 310 additions and 512 deletions
+1 -3
View File
@@ -1411,8 +1411,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
| "get more out of gbrain", "is my brain set up right", "weekly brain checkup", "advise me on my brain", "gbrain advisor" | `skills/gbrain-advisor/SKILL.md` |
| Save or load reports | `skills/reports/SKILL.md` |
| "Create a skill", "improve this skill" | `skills/skill-creator/SKILL.md` |
| "save this learning to the vault", "capture this skill in Obsidian", "record this workflow in my notes", "put this setup change in the vault" | `skills/skill-vault-capture-policy/SKILL.md` |
| "Skillify this", "is this a skill?", "make this proper", "add tests and evals for this" | `skills/skillify/SKILL.md` |
| "Skillify this", "is this a skill?", "make this proper" | `skills/skillify/SKILL.md` |
| "Compress my resolver", "AGENTS.md too large", "RESOLVER.md too big", "functional area dispatcher", "shrink routing table" | `skills/functional-area-resolver/SKILL.md` |
| "Is gbrain healthy?", morning health check, skillpack-check | `skills/skillpack-check/SKILL.md` |
| "harvest this skill into gbrain", "publish this skill to gbrain", "lift this skill upstream", "share this skill with other gbrain clients", "promote my skill to gbrain" | `skills/skillpack-harvest/SKILL.md` |
@@ -1430,7 +1429,6 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
| "Set up GBrain", first boot | `skills/setup/SKILL.md` |
| "Now what?", "fill my brain", "cold start", "bootstrap", "import my data", "what should I import first" | `skills/cold-start/SKILL.md` |
| "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
| "Connect Obsidian to gbrain", "import my vault to gbrain", "sync vault and gbrain", "embed gbrain after vault update", "is gbrain synced with my vault" | `skills/obsidian-gbrain-safe-index/SKILL.md` |
| Brain health check, maintenance run | `skills/maintain/SKILL.md` |
| "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
| "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |
+2 -3
View File
@@ -39,8 +39,8 @@
"skills/briefing",
"skills/citation-fixer",
"skills/concept-synthesis",
"skills/cron-scheduler",
"skills/cross-modal-review",
"skills/cron-scheduler",
"skills/daily-task-manager",
"skills/daily-task-prep",
"skills/data-research",
@@ -54,11 +54,10 @@
"skills/media-ingest",
"skills/meeting-ingestion",
"skills/minion-orchestrator",
"skills/obsidian-gbrain-safe-index",
"skills/perplexity-research",
"skills/query",
"skills/repo-architecture",
"skills/reports",
"skills/repo-architecture",
"skills/signal-detector",
"skills/skill-creator",
"skills/skillify",
+1 -3
View File
@@ -60,8 +60,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
| "get more out of gbrain", "is my brain set up right", "weekly brain checkup", "advise me on my brain", "gbrain advisor" | `skills/gbrain-advisor/SKILL.md` |
| Save or load reports | `skills/reports/SKILL.md` |
| "Create a skill", "improve this skill" | `skills/skill-creator/SKILL.md` |
| "save this learning to the vault", "capture this skill in Obsidian", "record this workflow in my notes", "put this setup change in the vault" | `skills/skill-vault-capture-policy/SKILL.md` |
| "Skillify this", "is this a skill?", "make this proper", "add tests and evals for this" | `skills/skillify/SKILL.md` |
| "Skillify this", "is this a skill?", "make this proper" | `skills/skillify/SKILL.md` |
| "Compress my resolver", "AGENTS.md too large", "RESOLVER.md too big", "functional area dispatcher", "shrink routing table" | `skills/functional-area-resolver/SKILL.md` |
| "Is gbrain healthy?", morning health check, skillpack-check | `skills/skillpack-check/SKILL.md` |
| "harvest this skill into gbrain", "publish this skill to gbrain", "lift this skill upstream", "share this skill with other gbrain clients", "promote my skill to gbrain" | `skills/skillpack-harvest/SKILL.md` |
@@ -79,7 +78,6 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
| "Set up GBrain", first boot | `skills/setup/SKILL.md` |
| "Now what?", "fill my brain", "cold start", "bootstrap", "import my data", "what should I import first" | `skills/cold-start/SKILL.md` |
| "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
| "Connect Obsidian to gbrain", "import my vault to gbrain", "sync vault and gbrain", "embed gbrain after vault update", "is gbrain synced with my vault" | `skills/obsidian-gbrain-safe-index/SKILL.md` |
| Brain health check, maintenance run | `skills/maintain/SKILL.md` |
| "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
| "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |
+1 -11
View File
@@ -2,7 +2,7 @@
"name": "gbrain",
"version": "0.32.3.0",
"conformance_version": "1.0.0",
"description": "Personal knowledge brain with hybrid RAG search GStack mod for agent platforms",
"description": "Personal knowledge brain with hybrid RAG search \u2014 GStack mod for agent platforms",
"skills": [
{
"name": "ingest",
@@ -34,11 +34,6 @@
"path": "migrate/SKILL.md",
"description": "Universal migration from Obsidian, Notion, Logseq, markdown, CSV, JSON, Roam"
},
{
"name": "obsidian-gbrain-safe-index",
"path": "obsidian-gbrain-safe-index/SKILL.md",
"description": "Connect and sync an Obsidian-style Markdown vault with gbrain using a cost-controlled import-first workflow, explicit embedding gates, and durable skill/workflow capture."
},
{
"name": "setup",
"path": "setup/SKILL.md",
@@ -268,11 +263,6 @@
"name": "skill-optimizer",
"path": "skill-optimizer/SKILL.md",
"description": "Self-evolving skill optimization via gbrain skillopt — SkillOpt-paper-grounded text-space optimizer with validation gating (median-of-3 + epsilon=0.05), bundled-skill safety, bootstrap review sentinel, per-skill DB lock, and atomic versioned writes."
},
{
"name": "skill-vault-capture-policy",
"path": "skill-vault-capture-policy/SKILL.md",
"description": "Capture durable operational learnings, new agent skills, and environment/setup changes into the Obsidian vault instead of transient chat memory."
}
],
"dependencies": {
-137
View File
@@ -1,137 +0,0 @@
---
name: obsidian-gbrain-safe-index
version: 1.0.0
description: |
Connect, maintain, and sync an Obsidian-style Markdown vault with gbrain while preserving a cost-controlled workflow: the vault remains the source of truth, gbrain is the searchable/embedded index, durable skills/workflows are captured into the vault, and paid embedding runs only after explicit approval.
triggers:
- "connect Obsidian to gbrain"
- "import my vault to gbrain"
- "sync vault and gbrain"
- "capture this skill in my vault"
- "embed gbrain after vault update"
- "is gbrain synced with my vault"
tools:
- terminal
- read_file
- search_files
- write_file
- patch
mutating: true
---
# Obsidian → gbrain Safe Index and Capture
## Contract
This skill guarantees:
- Treats the user's Obsidian-style Markdown vault as the source of truth before gbrain indexing.
- Keeps gbrain in conservative mode unless the user explicitly approves a more expensive mode.
- Imports vault changes with `--no-embed` first, then embeds only after explicit approval for the specific paid action.
- Captures durable new skills, workflows, and environment learnings into the vault instead of leaving them only in chat memory.
- Verifies every sync with concrete `gbrain stats`, search mode, and, when embeddings run, exact embedded chunk counts.
## Phases
1. **Resolve the vault path.**
- Prefer an existing environment variable such as `OBSIDIAN_VAULT_PATH` or `WIKI_PATH`.
- If no path is configured, search likely note directories and ask the user before writing.
- Verify the directory exists and contains markdown files or an `.obsidian` directory.
2. **Read vault operating rules before writing.**
- If the vault has `SCHEMA.md`, `index.md`, `log.md`, `AGENTS.md`, or similar operating files, read them before ingest/query/major edit.
- Respect immutable source folders such as `raw/` when the vault declares them.
- Use the vault's native link convention, usually Obsidian `[[wikilinks]]`, for durable relationships.
3. **MECE/capture decision.**
- If new knowledge belongs on an existing page, update that page.
- If it is a distinct recurring workflow or operational policy, create a small meta or concept page following the vault schema.
- Update the vault index/catalog for every new page when the vault maintains one.
- Append a log entry for meaningful vault updates when the vault maintains a log.
4. **Safe gbrain import path.**
- Pre-check source directory; do not import a nonexistent path.
- Run `gbrain config set search.mode conservative` before/after risky reinit steps.
- Run `gbrain import "$OBSIDIAN_VAULT_PATH" --no-embed`.
- Run `gbrain extract links --source fs --dir "$OBSIDIAN_VAULT_PATH"` when wikilinks changed materially.
5. **Paid embedding gate.**
- Do not run `gbrain embed --stale` unless the user explicitly asks or a prior instruction clearly approved this exact paid action.
- Before embedding, verify provider readiness with `gbrain providers test --model <provider:model>`.
- Confirm the configured embedding dimensions match the local schema.
- After embedding, verify `gbrain stats` and record exact `Pages`, `Chunks`, `Embedded`, and `Links` counts.
6. **Final verification and vault echo.**
- Run `gbrain stats` and `gbrain search modes`.
- If vault files changed, re-import with `--no-embed`; if embedding was approved, embed stale chunks afterward.
- Report what changed, what was free/local, what used API billing, and what remains pending.
## Output Format
Use a compact status table:
| Item | Status |
|---|---|
| Vault path | `/path` |
| Vault updated | yes/no + files |
| gbrain mode | conservative/balanced/tokenmax |
| Import | `--no-embed` completed / skipped / failed |
| Pages/chunks | exact counts from `gbrain stats` |
| Embeddings | exact count; note whether this run used API billing |
| Links | exact count |
| Background jobs | none / list exact jobs |
Then include:
- **Safe next step:** free/local action.
- **Paid next step:** embedding/LLM action, if any, with explicit approval requirement.
## Anti-Patterns
- Creating a duplicate skill/page when an existing Obsidian, gbrain, or vault-ingest skill already covers the workflow.
- Running `gbrain embed --stale`, `gbrain dream`, `gbrain autopilot --install`, `gbrain onboard --auto`, or `tokenmax` without explicit cost approval.
- Importing a nonexistent or wrong directory and treating a zero-page import as success.
- Forgetting to update the vault index/catalog and log after creating or materially updating vault pages.
- Recording API keys, tokens, or raw secrets in the vault or final response.
## Tools Used
- `read_file` — read vault schema/index/log and target notes.
- `search_files` — find existing vault pages and avoid duplicates.
- `write_file` / `patch` — create or update vault pages.
- `terminal` — run `gbrain`, `git`, and environment checks with secret values redacted.
## Safe Commands
```bash
export PATH="$HOME/.bun/bin:$PATH"
gbrain config set search.mode conservative
gbrain import "$OBSIDIAN_VAULT_PATH" --no-embed
gbrain extract links --source fs --dir "$OBSIDIAN_VAULT_PATH"
gbrain stats
gbrain search modes
```
## Paid / Approval-Gated Commands
```bash
gbrain providers test --model <provider:model>
gbrain embed --stale
gbrain dream
gbrain autopilot --install
gbrain onboard --auto --max-usd 5
gbrain config set search.mode tokenmax
```
## Verification Checklist
- [ ] Vault path exists and is the intended source.
- [ ] Vault operating files were read before edits when present.
- [ ] Existing pages/skills were searched to avoid duplicates.
- [ ] New/updated vault pages follow the vault schema and link convention.
- [ ] Vault index/catalog updated for new pages when present.
- [ ] Vault log appended for meaningful actions when present.
- [ ] `gbrain import ... --no-embed` completed.
- [ ] `gbrain stats` recorded pages/chunks/embeddings/links.
- [ ] `gbrain search modes` confirms conservative mode unless a different mode was explicitly approved.
- [ ] No paid/background commands ran without approval.
@@ -1,11 +0,0 @@
// Routing eval fixtures for skills/obsidian-gbrain-safe-index.
// Positive cases: intents embed a trigger phrase in natural surrounding context
// (never verbatim-identical to a trigger — the fixture linter rejects tautologies).
{"intent": "help me connect Obsidian to gbrain for my notes", "expected_skill": "obsidian-gbrain-safe-index"}
{"intent": "import my vault to gbrain but skip embeddings for now", "expected_skill": "obsidian-gbrain-safe-index"}
{"intent": "please sync vault and gbrain after I edit notes", "expected_skill": "obsidian-gbrain-safe-index"}
{"intent": "run embed gbrain after vault update tonight", "expected_skill": "obsidian-gbrain-safe-index"}
{"intent": "hey is gbrain synced with my vault right now", "expected_skill": "obsidian-gbrain-safe-index"}
// Negative cases: related but owned by other skills. Assert NO route to this skill.
{"intent": "migrate my notes from Notion to gbrain", "expected_skill": null}
{"intent": "what is on my calendar tomorrow", "expected_skill": null}
-100
View File
@@ -1,100 +0,0 @@
---
name: skill-vault-capture-policy
version: 1.0.0
description: |
Use when the user wants a durable operational learning, new agent skill, or
important environment/setup change to be captured into the Obsidian vault
instead of left only in transient chat memory. Covers the capture rule,
preferred page patterns, and index/log update obligations.
triggers:
- "save this learning to the vault"
- "capture this skill in Obsidian"
- "record this workflow in my notes"
- "put this setup change in the vault"
- "should we add this to the knowledge base"
tools:
- read_file
- search_files
- write_file
- patch
mutating: true
---
# Skill Vault Capture Policy
## Contract
This skill guarantees:
- Durable operational learnings, newly adopted skills, and important environment/setup changes are captured into the Obsidian vault rather than left only in chat memory.
- An existing page is updated when the knowledge clearly belongs there; a new page is created only when the topic is distinct and likely to recur.
- Every new vault page is added to `index.md`.
- Every meaningful create/update appends a dated entry to `log.md`.
- Small linked pages are preferred over one giant running note.
## Phases
1. **Classify the learning.**
- New gbrain operating rule, cost control, or embedding/provider change.
- New agent-fork / harness / coding-tool integration fact.
- New Obsidian vault workflow or structure decision.
- New recurring agent skill that changes how the agent should operate here.
2. **Avoid duplicates.**
- Search the vault for an existing page that already owns the topic.
- If found, update it with a new section or dated note rather than creating a near-duplicate.
3. **Create when distinct.**
- Place new pages under the vault schema: `_meta/` for operating notes, `concepts/` for workflows, `entities/` for tools/people.
- Use YAML frontmatter and at least two `[[wikilinks]]` unless it is a short seed page.
4. **Update navigation.**
- Add the page to `index.md` under the correct type heading.
- Append a `## [YYYY-MM-DD] create|update | subject` entry to `log.md`.
5. **Report the capture.**
- State which files changed and whether `index.md` / `log.md` were updated.
## Output Format
Use a short status block:
| Item | Status |
|---|---|
| Learning classified | type |
| Page created/updated | path |
| index.md updated | yes/no |
| log.md updated | yes/no |
## Anti-Patterns
- Leaving durable learnings only in chat memory.
- Creating a near-duplicate page instead of updating the existing one.
- Forgetting to update `index.md` and `log.md`.
- Writing one giant running note instead of small linked pages.
- Recording secrets, API keys, or raw credentials in the vault.
## Tools Used
- `read_file` — read `SCHEMA.md`, `index.md`, `log.md`, and target pages.
- `search_files` — find existing pages to avoid duplicates.
- `write_file` / `patch` — create or update vault pages and navigation.
## Safe Commands
```bash
# inspect vault navigation before writing
read SCHEMA.md index.md log.md
# create or update a page, then refresh catalog/log
# index.md: add [[page-slug]] under the matching type heading
# log.md: append ## [YYYY-MM-DD] create|update | subject
```
## Verification Checklist
- [ ] Vault schema/read files were checked before writing.
- [ ] Existing pages were searched to avoid duplicates.
- [ ] New/updated page has frontmatter and wikilinks.
- [ ] `index.md` updated for new pages.
- [ ] `log.md` appended for meaningful actions.
- [ ] No secrets or raw credentials were written.
@@ -1,10 +0,0 @@
// Routing eval fixtures for skills/skill-vault-capture-policy.
// Positive cases: intents embed a trigger phrase in natural surrounding context
// (never verbatim-identical to a trigger — the fixture linter rejects tautologies).
{"intent": "please save this learning to the vault so we keep it", "expected_skill": "skill-vault-capture-policy", "ambiguous_with": ["idea-ingest"]}
{"intent": "we should capture this skill in Obsidian for reuse", "expected_skill": "skill-vault-capture-policy", "ambiguous_with": ["capture"]}
{"intent": "can you record this workflow in my notes for next time", "expected_skill": "skill-vault-capture-policy"}
{"intent": "put this setup change in the vault before we forget", "expected_skill": "skill-vault-capture-policy"}
// Negative cases: related but owned by other skills or out of scope.
{"intent": "connect my Obsidian vault to gbrain", "expected_skill": null}
{"intent": "what is on my calendar tomorrow", "expected_skill": null}
+22 -3
View File
@@ -433,7 +433,10 @@ export async function extractLinksFromFile(
async resolve(name: string, dirHint?: string | string[]): Promise<string | null> {
if (!name) return null;
const trimmed = name.trim();
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
// Same broadened slug-shape as makeResolver step 1: accepts
// digit-leading folders (`90-people/nicolai`) and nested paths.
// Exact Set membership guards it — no false positives.
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
return trimmed;
}
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
@@ -582,6 +585,17 @@ export interface ExtractOpts {
* before (single-'default'-source brains unaffected).
*/
sourceId?: string;
/**
* v0.42 — also extract frontmatter links on the incremental (slugs) path.
* `extractForSlugs` extracts BODY links only by default; set this true to also
* parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh
* when YAML is edited externally and synced in. Applied PER changed page, so the
* incremental walk stays bounded (no switch to a full DB scan). Only honored on
* the incremental path (`slugs` defined); the full-walk path already covers
* frontmatter via its own dispatch. Gated upstream by the config key
* `autopilot.incremental_extract_include_frontmatter` (default off).
*/
includeFrontmatter?: boolean;
}
/**
@@ -620,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr
// Nothing changed — skip entirely.
return result;
}
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId);
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter);
result.links_created = r.links_created;
result.timeline_entries_created = r.timeline_created;
result.pages_processed = r.pages;
@@ -1011,6 +1025,11 @@ async function extractForSlugs(
signal?: AbortSignal,
// #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId).
sourceId?: string,
// v0.42: when true, also extract frontmatter links per changed page so
// externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle.
// Default false preserves the body-only incremental behavior. Gated upstream
// by `autopilot.incremental_extract_include_frontmatter`.
includeFrontmatter: boolean = false,
): Promise<{ links_created: number; timeline_created: number; pages: number }> {
// Build the full slug set for link resolution (fast: just readdir, no file reads)
const allFiles = walkMarkdownFiles(brainDir);
@@ -1085,7 +1104,7 @@ async function extractForSlugs(
const content = readFileSync(fullPath, 'utf-8');
if (doLinks) {
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename });
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter });
for (const link of links) {
if (dryRun) {
if (!jsonMode) console.log(` ${link.from_slug}${link.to_slug} (${link.link_type})`);
+12
View File
@@ -121,6 +121,18 @@ export interface GBrainConfig {
/** Daily spend cap (USD); bounds drains/day = floor(cap / ~$0.30). Default 2.0. */
max_usd_per_day?: number;
};
/**
* v0.42 — keep frontmatter links fresh on the incremental cycle. The cycle's
* extract phase re-extracts only the slugs a sync changed, but `extractForSlugs`
* extracts BODY links only — frontmatter (`sources:`/`related:` etc.) link edges
* silently drift stale when a page's YAML is edited externally and synced in.
* Set true to also extract frontmatter links per changed page each cycle, keeping
* externally-edited YAML edges fresh without a full rescan. Default false
* (preserves current behavior). Read via the file/env/DB plane in the cycle's
* extract dispatch. Disable/enable with
* `gbrain config set autopilot.incremental_extract_include_frontmatter <bool>`.
*/
incremental_extract_include_frontmatter?: boolean;
};
eval?: {
/** false disables capture entirely. Defaults to true. */
+16
View File
@@ -996,6 +996,21 @@ async function runPhaseExtract(
): Promise<PhaseResult> {
try {
const { runExtractCore } = await import('../commands/extract.ts');
const { loadConfig } = await import('./config.ts');
// Default off: the incremental cycle extracts body links only unless the
// operator opts in to keeping externally-edited frontmatter links fresh too.
// Both planes, file wins (env > file > DB precedence, per loadConfigWithEngine):
// `gbrain config set autopilot.incremental_extract_include_frontmatter true`
// writes the DB plane (engine.setConfig), so a file-plane-only read here
// would make the documented enable command a silent no-op (#2120 class).
const fileVal = loadConfig()?.autopilot?.incremental_extract_include_frontmatter;
let includeFrontmatter = fileVal === true;
if (fileVal === undefined) {
try {
includeFrontmatter =
(await engine.getConfig('autopilot.incremental_extract_include_frontmatter')) === 'true';
} catch { /* config table unreadable → default off */ }
}
// Extract is read-mostly against the filesystem + write to links table.
// Honor dryRun by skipping with a 'skipped' entry: extract doesn't have
// a clean dry-run mode today and runCycle should be honest about it.
@@ -1016,6 +1031,7 @@ async function runPhaseExtract(
slugs: changedSlugs, // undefined = full walk (first run / manual)
signal,
sourceId,
includeFrontmatter, // honored on the incremental (slugs) path only
});
const linksCreated = result?.links_created ?? 0;
const timelineCreated = result?.timeline_entries_created ?? 0;
+36 -3
View File
@@ -942,8 +942,17 @@ export function makeResolver(
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
// Step 1: already a slug? (dir/name shape, lowercase, hyphenated)
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed)) {
// Step 1: already a slug? Try an exact page lookup for any slug-shaped
// value (contains '/', slug charset). Broadened beyond the original
// single-segment lowercase-leading form (`^[a-z][a-z0-9-]*\/[a-z0-9]...`)
// to also accept digit-leading folders (`90-people/nicolai`,
// `01-trading/...`) and nested paths (`a/b/c`) — common in PARA-numbered
// vaults. This is an EXACT getPage match only — no fuzzy — so it never
// produces a false positive; a non-existent slug just falls through to
// the steps below. Fixes frontmatter `related: [[dir/slug]]` values
// (unwrapped by unwrapWikilink) that name a real page the strict regex
// could not reach and whose full-path fuzzy score is below threshold.
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) {
const page = await engine.getPage(trimmed);
if (page) {
cache.set(cacheKey, trimmed);
@@ -1003,6 +1012,25 @@ export function makeResolver(
// ─── Frontmatter extractor ──────────────────────────────────────
/**
* Unwrap an Obsidian `[[wikilink]]` frontmatter value to its bare link
* target so the resolver (which expects bare titles / dir slugs) can match
* it. Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`;
* without this, the resolver treats the brackets as part of the value and a
* `[[90-people/nicolai]]` is normalized into `90peoplenicolai`, so it never
* resolves. Strips a trailing `|alias`, `#heading`, or `^block` suffix — the
* link target only. The regex is anchored to a wholly-wrapped value
* (`^\s*\[\[…\]\]\s*$`), so bare titles and any value not fully wrapped pass
* through unchanged and existing behavior is preserved exactly.
*/
export function unwrapWikilink(value: string): string {
const match = /^\s*\[\[(.+?)\]\]\s*$/.exec(value);
if (!match) return value;
// Take the link target: drop |alias, then #heading / ^block suffixes.
const target = match[1].split('|')[0].split('#')[0].split('^')[0];
return target.trim();
}
export interface UnresolvedFrontmatterRef {
/** The frontmatter field name. */
field: string;
@@ -1060,7 +1088,12 @@ export async function extractFrontmatterLinks(
}
if (!name) continue; // skip numbers, nulls, malformed objects
const resolved = await resolver.resolve(name, mapping.dirHint);
// Accept Obsidian `[[wikilink]]` values in frontmatter link fields by
// unwrapping to the bare target before resolution. Bare titles pass
// through unchanged; the original `name` is preserved for the
// unresolved report and edge context.
const linkTarget = unwrapWikilink(name);
const resolved = await resolver.resolve(linkTarget, mapping.dirHint);
if (!resolved) {
unresolved.push({ field, name });
continue;
@@ -1,61 +0,0 @@
/**
* E2E smoke for skills/obsidian-gbrain-safe-index.
*
* Verifies the from-trigger-to-side-effect path that skillify requires:
* a real user trigger phrase routes to the skill, the resolver/check
* pipeline treats it as reachable, and the skill file exposes the
* gbrain commands the workflow actually runs.
*
* This stays local-only (no paid embedding, no external API): it asserts
* the documented safe/import path is present and parseable, not that it
* mutates a live brain.
*/
import { describe, expect, it } from 'bun:test';
import { existsSync, readFileSync } from 'fs';
import { join } from 'path';
const SKILLS = join(import.meta.dir, '..', '..', 'skills');
const SKILL_MD = join(SKILLS, 'obsidian-gbrain-safe-index', 'SKILL.md');
const RESOLVER = join(SKILLS, 'RESOLVER.md');
const TRIGGER_PHRASES = [
'connect my Obsidian vault to gbrain',
'import my vault to gbrain',
'sync vault and gbrain',
'capture this skill in my vault',
'embed gbrain after vault update',
'is gbrain synced with my vault',
];
describe('obsidian-gbrain-safe-index E2E', () => {
it('resolver maps real trigger phrasings to the skill', () => {
const resolver = readFileSync(RESOLVER, 'utf-8');
expect(resolver).toContain('obsidian-gbrain-safe-index/SKILL.md');
// Each representative phrase shares a token substring with a resolver row.
const rows = resolver
.split('\n')
.filter((l) => l.includes('obsidian-gbrain-safe-index/SKILL.md'))
.join('\n');
for (const phrase of TRIGGER_PHRASES) {
const hit = phrase
.toLowerCase()
.split(/\s+/)
.some((tok) => tok.length > 3 && rows.toLowerCase().includes(tok));
expect(hit, `no resolver token for: ${phrase}`).toBe(true);
}
});
it('skill documents the gbrain import-first safe path', () => {
const body = readFileSync(SKILL_MD, 'utf-8');
expect(body).toContain('gbrain import');
expect(body).toContain('--no-embed');
expect(body).toContain('gbrain config set search.mode conservative');
// Paid gate must be explicit, not a silent default.
expect(body).toContain('gbrain embed --stale');
});
it('skill is reachable from the skill tree', () => {
expect(existsSync(SKILL_MD)).toBe(true);
});
});
@@ -1,52 +0,0 @@
/**
* E2E smoke for skills/skill-vault-capture-policy.
*
* Verifies the from-trigger-to-side-effect path: a real capture request
* routes to the skill, and the skill documents the vault navigation
* obligations (index.md / log.md) that make a capture durable.
*
* Local-only: it asserts documented behavior, not live vault writes.
*/
import { describe, expect, it } from 'bun:test';
import { existsSync, readFileSync } from 'fs';
import { join } from 'path';
const SKILLS = join(import.meta.dir, '..', '..', 'skills');
const SKILL_MD = join(SKILLS, 'skill-vault-capture-policy', 'SKILL.md');
const RESOLVER = join(SKILLS, 'RESOLVER.md');
const TRIGGER_PHRASES = [
'save this learning to the vault',
'capture this skill in Obsidian',
'record this workflow in my notes',
'put this setup change in the knowledge base',
];
describe('skill-vault-capture-policy E2E', () => {
it('resolver maps real capture phrasings to the skill', () => {
const resolver = readFileSync(RESOLVER, 'utf-8');
expect(resolver).toContain('skill-vault-capture-policy/SKILL.md');
const rows = resolver
.split('\n')
.filter((l) => l.includes('skill-vault-capture-policy/SKILL.md'))
.join('\n');
for (const phrase of TRIGGER_PHRASES) {
const hit = phrase
.toLowerCase()
.split(/\s+/)
.some((tok) => tok.length > 3 && rows.toLowerCase().includes(tok));
expect(hit, `no resolver token for: ${phrase}`).toBe(true);
}
});
it('skill documents index.md and log.md update obligations', () => {
const body = readFileSync(SKILL_MD, 'utf-8');
expect(body).toContain('index.md');
expect(body).toContain('log.md');
});
it('skill is reachable from the skill tree', () => {
expect(existsSync(SKILL_MD)).toBe(true);
});
});
+34
View File
@@ -191,3 +191,37 @@ describe('runExtractCore — incremental cycle path (#417)', () => {
expect(result.links_created).toBeGreaterThan(0);
});
});
describe('runExtractCore — incremental frontmatter gate (includeFrontmatter)', () => {
// alice has a `source:` frontmatter edge but NO body links. The incremental
// path extracts body links only by default, so the frontmatter edge is the
// sole signal that distinguishes the gate off vs on.
const aliceFm = '---\nsource: companies/acme-example\n---\n# alice';
test('9. default (flag omitted) does NOT extract frontmatter links on the incremental path', async () => {
await seedPage('companies/acme-example', '# acme');
await seedPage('people/alice-example', aliceFm);
const result = await runExtractCore(engine as unknown as BrainEngine, {
mode: 'all',
dir: tempDir,
slugs: ['people/alice-example'],
});
// alice's only potential edge is her frontmatter `source:`; with the gate off
// it must not be extracted (preserves the body-only incremental behavior).
expect(result.pages_processed).toBe(1);
expect(result.links_created).toBe(0);
});
test('10. includeFrontmatter: true extracts the frontmatter link on the incremental path', async () => {
await seedPage('companies/acme-example', '# acme');
await seedPage('people/alice-example', aliceFm);
const result = await runExtractCore(engine as unknown as BrainEngine, {
mode: 'all',
dir: tempDir,
slugs: ['people/alice-example'],
includeFrontmatter: true,
});
// Same page, gate on → the `source:` frontmatter edge is now extracted.
expect(result.pages_processed).toBe(1);
expect(result.links_created).toBeGreaterThan(0);
});
});
+12
View File
@@ -76,6 +76,18 @@ describe('extractLinksFromFile', () => {
}
});
it('resolves wrapped [[wikilink]] digit-leading slug-path in frontmatter (fs resolver, broadened step 1)', async () => {
// Same bug class as makeResolver step 1 (#1983): the fs resolver's strict
// `^[a-z]…` slug regex rejected digit-leading / nested paths, so a PARA-vault
// `related: "[[90-people/nicolai]]"` never resolved even though the page exists.
const content = '---\nrelated: "[[90-people/nicolai]]"\ntype: concept\n---\nContent.';
const allSlugs = new Set(['wiki/note', '90-people/nicolai']);
const links = await extractLinksFromFile(content, 'wiki/note.md', allSlugs, { includeFrontmatter: true });
const related = links.filter(l => l.link_type === 'related_to');
expect(related).toHaveLength(1);
expect(related[0].to_slug).toBe('90-people/nicolai');
});
it('frontmatter extraction is default OFF (back-compat)', async () => {
// Without includeFrontmatter, fs-source no longer auto-extracts frontmatter.
// Matches db-source behavior. User opts in with --include-frontmatter flag.
+173
View File
@@ -9,6 +9,7 @@ import {
parseTimelineEntries,
isAutoLinkEnabled,
FRONTMATTER_LINK_MAP,
unwrapWikilink,
type SlugResolver,
} from '../src/core/link-extraction.ts';
import type { BrainEngine } from '../src/core/engine.ts';
@@ -1291,3 +1292,175 @@ describe('parseTimelineEntries — Format 3: inline [Source: ..., YYYY-MM-DD] ci
expect(parseTimelineEntries('[Source: import batch, 2025-07-01]')).toHaveLength(0);
});
});
// ─── Frontmatter [[wikilink]] + slug-path resolution ──────────────────────
// Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`,
// and PARA-numbered vaults use digit-leading / nested slug paths like
// `[[90-people/nicolai]]`. Both were silently dropped: brackets were treated
// as part of the value and the step-1 slug regex (`^[a-z]…`) rejected
// digit-leading / nested paths, while full-path fuzzy scored below threshold.
// Fix: unwrapWikilink() before resolution + an exact getPage() for any
// slug-shaped value (exact-match only → no false positives).
describe('unwrapWikilink', () => {
test('wrapped title → bare title', () => {
expect(unwrapWikilink('[[Monday Range]]')).toBe('Monday Range');
});
test('wrapped slug-path (digit-leading folder) → bare slug', () => {
expect(unwrapWikilink('[[90-people/nicolai]]')).toBe('90-people/nicolai');
});
test('wrapped nested slug-path → bare slug', () => {
expect(unwrapWikilink('[[01-trading/wiki/strategies/opening-range-breakout]]'))
.toBe('01-trading/wiki/strategies/opening-range-breakout');
});
test('strips |alias', () => {
expect(unwrapWikilink('[[90-people/nicolai|Nicolai]]')).toBe('90-people/nicolai');
});
test('strips #heading', () => {
expect(unwrapWikilink('[[Page#Section]]')).toBe('Page');
});
test('strips ^block', () => {
expect(unwrapWikilink('[[Page^abc123]]')).toBe('Page');
});
test('surrounding whitespace tolerated', () => {
expect(unwrapWikilink(' [[Page]] ')).toBe('Page');
});
test('bare title passes through unchanged', () => {
expect(unwrapWikilink('Monday Range')).toBe('Monday Range');
});
test('bare slug passes through unchanged', () => {
expect(unwrapWikilink('90-people/nicolai')).toBe('90-people/nicolai');
});
test('partially-wrapped value is NOT unwrapped (anchored)', () => {
// Not a wholly-wrapped value → left intact so existing behavior is exact.
expect(unwrapWikilink('see [[Page]] for detail')).toBe('see [[Page]] for detail');
});
});
describe('makeResolver — slug-path exact getPage (step 1 broadened)', () => {
function fakeEngine(
slugs: string[],
fuzzyMap: Map<string, { slug: string; similarity: number }> = new Map(),
): BrainEngine {
const lookup = new Set(slugs);
return {
async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; },
async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; },
async searchKeyword() { return []; },
} as unknown as BrainEngine;
}
test('digit-leading folder slug resolves via exact getPage', async () => {
const r = makeResolver(fakeEngine(['90-people/nicolai']));
expect(await r.resolve('90-people/nicolai')).toBe('90-people/nicolai');
});
test('nested (>2 segment) slug resolves via exact getPage', async () => {
const r = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout']));
expect(await r.resolve('01-trading/wiki/strategies/opening-range-breakout'))
.toBe('01-trading/wiki/strategies/opening-range-breakout');
});
test('regression: single-segment lowercase slug still resolves', async () => {
const r = makeResolver(fakeEngine(['people/pedro']));
expect(await r.resolve('people/pedro')).toBe('people/pedro');
});
test('exact-only: slug-shaped value with no matching page falls through (no false positive)', async () => {
// `90-people/ghost` is slug-shaped but absent → step-1 getPage misses,
// no fuzzy hit → null. Never invents an edge.
const r = makeResolver(fakeEngine(['90-people/nicolai']));
expect(await r.resolve('90-people/ghost')).toBeNull();
});
test('non-slug value still routes to fuzzy', async () => {
const r = makeResolver(fakeEngine(
['01-trading/monday-range'],
new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]),
));
expect(await r.resolve('Monday Range')).toBe('01-trading/monday-range');
});
});
describe('extractFrontmatterLinks — [[wikilink]] related: values (end-to-end)', () => {
function fakeEngine(
slugs: string[],
fuzzyMap: Map<string, { slug: string; similarity: number }> = new Map(),
): BrainEngine {
const lookup = new Set(slugs);
return {
async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; },
async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; },
async searchKeyword() { return []; },
} as unknown as BrainEngine;
}
test('wrapped slug-path related: resolves (the core win)', async () => {
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
const { candidates, unresolved } = await extractFrontmatterLinks(
'wiki/originals/ideas/note', 'note' as never,
{ related: '[[90-people/nicolai]]' }, resolver,
);
expect(unresolved).toHaveLength(0);
expect(candidates).toHaveLength(1);
expect(candidates[0]).toMatchObject({
fromSlug: 'wiki/originals/ideas/note',
targetSlug: '90-people/nicolai',
linkType: 'related_to',
linkSource: 'frontmatter',
});
});
test('wrapped nested slug-path related: resolves', async () => {
const resolver = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout']));
const { candidates } = await extractFrontmatterLinks(
'wiki/note', 'note' as never,
{ related: ['[[01-trading/wiki/strategies/opening-range-breakout]]'] }, resolver,
);
expect(candidates).toHaveLength(1);
expect(candidates[0].targetSlug).toBe('01-trading/wiki/strategies/opening-range-breakout');
});
test('wrapped value with |alias resolves to the target', async () => {
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
const { candidates } = await extractFrontmatterLinks(
'wiki/note', 'note' as never,
{ related: '[[90-people/nicolai|Nicolai]]' }, resolver,
);
expect(candidates).toHaveLength(1);
expect(candidates[0].targetSlug).toBe('90-people/nicolai');
});
test('regression: bare slug related: still resolves', async () => {
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
const { candidates } = await extractFrontmatterLinks(
'wiki/note', 'note' as never,
{ related: '90-people/nicolai' }, resolver,
);
expect(candidates).toHaveLength(1);
expect(candidates[0].targetSlug).toBe('90-people/nicolai');
});
test('regression: wrapped title resolves via fuzzy (brackets harmless)', async () => {
const resolver = makeResolver(fakeEngine(
['01-trading/monday-range'],
new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]),
));
const { candidates } = await extractFrontmatterLinks(
'wiki/note', 'note' as never,
{ related: '[[Monday Range]]' }, resolver,
);
expect(candidates).toHaveLength(1);
expect(candidates[0].targetSlug).toBe('01-trading/monday-range');
});
test('unknown wrapped slug → unresolved (no crash), original value preserved', async () => {
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
const { candidates, unresolved } = await extractFrontmatterLinks(
'wiki/note', 'note' as never,
{ related: '[[99-archive/does-not-exist]]' }, resolver,
);
expect(candidates).toHaveLength(0);
expect(unresolved).toHaveLength(1);
expect(unresolved[0]).toEqual({ field: 'related', name: '[[99-archive/does-not-exist]]' });
});
});
-51
View File
@@ -1,51 +0,0 @@
import { describe, expect, it } from 'bun:test';
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
const SKILL_DIR = join(import.meta.dir, '..', 'skills', 'obsidian-gbrain-safe-index');
const SKILL_MD = join(SKILL_DIR, 'SKILL.md');
const RESOLVER = join(import.meta.dir, '..', 'skills', 'RESOLVER.md');
function parseFrontmatter(raw: string): Record<string, unknown> {
const m = raw.match(/^---\n([\s\S]*?)\n---/);
if (!m) throw new Error('no frontmatter');
const out: Record<string, unknown> = {};
for (const line of m[1].split('\n')) {
const mm = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
if (mm) out[mm[1]] = mm[2].trim();
}
return out;
}
describe('obsidian-gbrain-safe-index skill', () => {
it('has a SKILL.md with required frontmatter', () => {
expect(existsSync(SKILL_MD)).toBe(true);
const fm = parseFrontmatter(readFileSync(SKILL_MD, 'utf-8'));
expect(fm['name']).toBe('obsidian-gbrain-safe-index');
expect(fm['description']).toBeTruthy();
});
it('has the required conformance sections', () => {
const body = readFileSync(SKILL_MD, 'utf-8');
for (const section of ['## Contract', '## Phases', '## Output Format', '## Anti-Patterns']) {
expect(body.includes(section), `missing ${section}`).toBe(true);
}
});
it('is registered in RESOLVER.md', () => {
expect(existsSync(RESOLVER)).toBe(true);
const resolver = readFileSync(RESOLVER, 'utf-8');
expect(resolver.includes('obsidian-gbrain-safe-index/SKILL.md')).toBe(true);
});
it('has routing-eval fixtures that exercise real trigger phrasings', () => {
const evalPath = join(SKILL_DIR, 'routing-eval.jsonl');
expect(existsSync(evalPath)).toBe(true);
const lines = readFileSync(evalPath, 'utf-8')
.split('\n')
.filter((l) => l.trim() && !l.trim().startsWith('//'))
.map((l) => JSON.parse(l));
const positives = lines.filter((l) => l.expected_skill === 'obsidian-gbrain-safe-index');
expect(positives.length).toBeGreaterThanOrEqual(5);
});
});
-64
View File
@@ -1,64 +0,0 @@
import { describe, expect, it } from 'bun:test';
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
import { DEFAULT_PRIVATE_PATTERNS } from '../src/core/skillpack/harvest-lint.ts';
const SKILL_DIR = join(import.meta.dir, '..', 'skills', 'skill-vault-capture-policy');
const SKILL_MD = join(SKILL_DIR, 'SKILL.md');
const RESOLVER = join(import.meta.dir, '..', 'skills', 'RESOLVER.md');
function parseFrontmatter(raw: string): Record<string, unknown> {
const m = raw.match(/^---\n([\s\S]*?)\n---/);
if (!m) throw new Error('no frontmatter');
const out: Record<string, unknown> = {};
for (const line of m[1].split('\n')) {
const mm = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
if (mm) out[mm[1]] = mm[2].trim();
}
return out;
}
describe('skill-vault-capture-policy skill', () => {
it('has a SKILL.md with required frontmatter', () => {
expect(existsSync(SKILL_MD)).toBe(true);
const fm = parseFrontmatter(readFileSync(SKILL_MD, 'utf-8'));
expect(fm['name']).toBe('skill-vault-capture-policy');
expect(fm['description']).toBeTruthy();
});
it('has the required conformance sections', () => {
const body = readFileSync(SKILL_MD, 'utf-8');
for (const section of ['## Contract', '## Phases', '## Output Format', '## Anti-Patterns']) {
expect(body.includes(section), `missing ${section}`).toBe(true);
}
});
it('is registered in RESOLVER.md', () => {
expect(existsSync(RESOLVER)).toBe(true);
expect(readFileSync(RESOLVER, 'utf-8').includes('skill-vault-capture-policy/SKILL.md')).toBe(true);
});
it('contains no private user or agent-fork names (privacy rule)', () => {
for (const file of [SKILL_MD, join(SKILL_DIR, 'routing-eval.jsonl')]) {
const body = readFileSync(file, 'utf-8');
// DEFAULT_PRIVATE_PATTERNS[0] is the banned fork-name pattern; sourced
// from harvest-lint so this file never contains the literal itself
// (scripts/check-privacy.sh would reject it).
const forkName = new RegExp(DEFAULT_PRIVATE_PATTERNS[0], 'i');
for (const name of [/\bAdam\b/, /\bHermes\b/, /\bHerdr\b/, /\bArk\b/, forkName]) {
expect(name.test(body), `private name ${name} in ${file}`).toBe(false);
}
}
});
it('has routing-eval fixtures', () => {
const evalPath = join(SKILL_DIR, 'routing-eval.jsonl');
expect(existsSync(evalPath)).toBe(true);
const positives = readFileSync(evalPath, 'utf-8')
.split('\n')
.filter((l) => l.trim() && !l.trim().startsWith('//'))
.map((l) => JSON.parse(l))
.filter((l) => l.expected_skill === 'skill-vault-capture-policy');
expect(positives.length).toBeGreaterThanOrEqual(4);
});
});