Compare commits

...
Author SHA1 Message Date
root bdc71b42c4 feat: Phases 2+3 — graph, timeline, ingest, contradictions, confidence
Phase 2 (Relationships + Temporal):
- gbrain_graph: traverse entity relationships with depth control
- gbrain_timeline: temporal queries via git log + timeline entry parsing
- gbrain_ingest: create/update brain pages with auto-reindex

Phase 3 (Intelligence):
- gbrain_contradictions: detect numeric/factual conflicts across sources
- gbrain_confidence: score claims by source count, recency, corroboration

Also adds edge traversal methods to store.ts and registers all 7 tools
in index.ts. 1,153 new lines, compiles clean.
2026-04-07 16:18:10 +00:00
root 93aa9d63fb feat: OpenClaw plugin — Phase 1 core search
OpenClaw plugin that makes GBrain semantically searchable via native agent tools.
Lives in openclaw-plugin/ alongside the main Postgres-based GBrain product.

Tools:
- gbrain_query: semantic search with scope filtering and compiled truth awareness
- gbrain_resolve: entity resolution via exact match, aliases, and embedding similarity

Infrastructure:
- Smart chunking: frontmatter+summary (high weight), compiled truth body, optional timeline
- Voyage API embeddings (1024-dim) with batch support
- SQLite storage with brute-force cosine similarity (portable, zero deps)
- Git-based incremental sync (polls HEAD every 30s)
- Background watcher service via api.registerService()
- CLI: openclaw gbrain status/reindex/query/resolve

Stack: TypeScript ESM, better-sqlite3, gray-matter, @sinclair/typebox
2026-04-07 15:29:56 +00:00
21 changed files with 2992 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
node_modules/
dist/
*.db
package-lock.json
+57
View File
@@ -0,0 +1,57 @@
{
"id": "gbrain",
"name": "GBrain",
"version": "0.1.0",
"description": "Personal knowledge brain with semantic search, entity resolution, and relationship traversal",
"author": "Garry Tan",
"license": "MIT",
"main": "dist/index.js",
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {
"brainPath": {
"type": "string",
"description": "Path to the markdown knowledge repository",
"default": "/data/brain"
},
"indexPath": {
"type": "string",
"description": "Path for the SQLite index database",
"default": "/data/db/gbrain.db"
},
"embeddingModel": {
"type": "string",
"description": "Embedding model to use (auto-detects from OpenClaw config if not set)",
"default": "auto"
},
"indexTimeline": {
"type": "boolean",
"description": "Whether to index timeline entries (below the line). Increases index size but enables temporal queries.",
"default": false
},
"watchInterval": {
"type": "number",
"description": "Seconds between git HEAD polls for incremental re-indexing",
"default": 30
},
"chunkMaxTokens": {
"type": "number",
"description": "Maximum tokens per chunk for embedding",
"default": 1000
},
"directories": {
"type": "array",
"items": { "type": "string" },
"description": "Directories to index (relative to brainPath). Empty = all.",
"default": []
},
"excludeDirectories": {
"type": "array",
"items": { "type": "string" },
"description": "Directories to skip (e.g. '.raw', 'archive')",
"default": [".raw", ".git", "node_modules"]
}
}
}
}
+32
View File
@@ -0,0 +1,32 @@
{
"name": "@garrytan/openclaw-gbrain",
"version": "0.1.0",
"description": "Personal knowledge brain plugin for OpenClaw — semantic search, entity resolution, and relationship traversal for markdown knowledge repos",
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"dev": "tsc --watch"
},
"openclaw": {
"pluginManifest": "openclaw.plugin.json",
"extensions": ["tools", "cli", "service"],
"compatVersions": {
"openclaw": ">=1.0.0"
}
},
"dependencies": {
"@sinclair/typebox": "^0.32.0",
"better-sqlite3": "^9.4.0",
"gray-matter": "^4.0.3"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.8",
"@types/node": "^20.0.0",
"typescript": "^5.3.0"
},
"keywords": ["openclaw", "plugin", "knowledge-base", "semantic-search", "markdown"],
"author": "Garry Tan",
"license": "MIT"
}
+166
View File
@@ -0,0 +1,166 @@
import type { CliContext } from "openclaw/plugin-sdk/plugin-entry";
import { fullReindex, incrementalReindex, getCurrentHead } from "./indexer/sync.js";
import type { GBrainStore } from "./indexer/store.js";
import type { GBrainConfig } from "./types/config.js";
import { executeQuery } from "./tools/query.js";
import { executeResolve } from "./tools/resolve.js";
export function registerCli(store: GBrainStore, config: GBrainConfig) {
return async ({ program }: CliContext): Promise<void> => {
const gbrain = program.command("gbrain").description("Knowledge brain management");
// openclaw gbrain status
gbrain.command("status")
.description("Show index stats and configuration")
.action(async () => {
const stats = store.getStats();
const head = getCurrentHead(config.brainPath);
console.log("\nGBrain Index Status");
console.log("═══════════════════════════════════════");
console.log(`Brain path: ${config.brainPath}`);
console.log(`Index path: ${config.indexPath}`);
console.log(`Pages indexed: ${stats.pageCount.toLocaleString()}`);
console.log(`Chunks: ${stats.chunkCount.toLocaleString()}`);
console.log(`Relationship edges: ${stats.edgeCount.toLocaleString()}`);
console.log(
`Index size: ${formatBytes(stats.indexSizeBytes)}`
);
console.log(`Last sync: ${stats.lastSync ?? "never"}`);
console.log(`Embedding model: ${stats.embeddingModel ?? "not set"}`);
console.log(`Current HEAD: ${head ?? "not a git repo"}`);
console.log(`Timeline indexed: ${config.indexTimeline ? "yes" : "no"}`);
console.log("");
});
// openclaw gbrain reindex [--full] [--dry-run]
gbrain.command("reindex")
.description("Rebuild the knowledge brain index")
.option("--full", "Force full reindex (ignore sync state)")
.option("--dry-run", "Show what would be reindexed without making changes")
.action(async (...args: unknown[]) => {
const opts = args[args.length - 1] as Record<string, unknown>;
const full = Boolean(opts["full"]);
const dryRun = Boolean(opts["dry-run"] ?? opts["dryRun"]);
if (dryRun) {
console.log("[gbrain] Dry run — no changes will be made.");
}
const lastHead = store.getSyncState("last_synced_head");
const currentHead = getCurrentHead(config.brainPath);
if (!full && lastHead && currentHead && lastHead !== currentHead) {
console.log(`[gbrain] Incremental reindex: ${lastHead.slice(0, 8)}${currentHead.slice(0, 8)}`);
const result = await incrementalReindex(
lastHead,
currentHead,
store,
config,
{
dryRun,
onProgress: (indexed, total, path) => {
if (indexed % 10 === 0 || indexed === total) {
console.log(`[gbrain] ${indexed}/${total}${path}`);
}
},
}
);
console.log(
`[gbrain] Done. Indexed ${result.indexed} files, deleted ${result.deleted}.`
);
} else {
if (!full) {
console.log("[gbrain] No prior sync state — running full reindex.");
} else {
console.log("[gbrain] Full reindex requested.");
}
const result = await fullReindex(store, config, {
dryRun,
onProgress: (indexed, total, path) => {
if (indexed % 100 === 0 || indexed === total) {
console.log(`[gbrain] ${indexed}/${total}${path}`);
}
},
});
console.log(
`[gbrain] Done. Indexed ${result.indexed} / ${result.total} files.`
);
}
});
// openclaw gbrain query <query>
gbrain.command("query")
.description("Run a semantic query against the brain")
.option("--scope <scope>", "Limit to directory (people, companies, etc.)")
.option("--limit <n>", "Max results", "5")
.action(async (...args: unknown[]) => {
const [queryStr, opts] = args as [string, Record<string, unknown>];
const apiKey = process.env["VOYAGE_API_KEY"] ?? "";
if (!apiKey) {
console.error("[gbrain] VOYAGE_API_KEY not set");
return;
}
const result = await executeQuery(
{
query: queryStr,
scope: opts["scope"] as "all" | undefined,
limit: parseInt(String(opts["limit"] ?? "5"), 10),
},
store,
apiKey
);
console.log(
`\n${result.results.length} results (${result.queryTimeMs}ms, ${result.totalIndexed} pages indexed)\n`
);
for (const r of result.results) {
console.log(`[${r.score.toFixed(3)}] ${r.title}${r.path}`);
console.log(` ${r.excerpt.replace(/\n/g, " ").slice(0, 120)}`);
console.log("");
}
});
// openclaw gbrain resolve <name>
gbrain.command("resolve")
.description("Resolve a name or reference to its brain page")
.action(async (...args: unknown[]) => {
const [nameStr] = args as [string];
const apiKey = process.env["VOYAGE_API_KEY"] ?? "";
const result = await executeResolve(
{ name: nameStr },
store,
apiKey
);
if (result.match) {
console.log(`\nResolved: ${result.match.title} (${result.match.confidence.toFixed(2)} confidence)`);
console.log(`Path: ${result.match.path}`);
console.log(`Match reason: ${result.match.matchReason}`);
if (result.match.aliases.length > 0) {
console.log(`Aliases: ${result.match.aliases.join(", ")}`);
}
console.log(`\n${result.match.excerpt}`);
} else {
console.log(`\nNo confident match found for "${nameStr}".`);
if (result.candidates.length > 0) {
console.log("Candidates:");
for (const c of result.candidates) {
console.log(
` [${c.confidence.toFixed(2)}] ${c.title}${c.path} (${c.matchReason})`
);
}
}
}
});
};
}
function formatBytes(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
+388
View File
@@ -0,0 +1,388 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { Type } from "@sinclair/typebox";
import { GBrainStore } from "./indexer/store.js";
import { resolveConfig } from "./types/config.js";
import { executeQuery } from "./tools/query.js";
import { executeResolve } from "./tools/resolve.js";
import { executeGraph } from "./tools/graph.js";
import { executeTimeline } from "./tools/timeline.js";
import { executeIngest } from "./tools/ingest.js";
import { executeContradictions } from "./tools/contradictions.js";
import { executeConfidence } from "./tools/confidence.js";
import { registerCli } from "./cli.js";
import { createWatcherService } from "./service.js";
definePluginEntry((api) => {
const config = resolveConfig(api.config);
const store = new GBrainStore(config.indexPath);
const apiKey = process.env["VOYAGE_API_KEY"] ?? "";
// ── Tools ────────────────────────────────────────────────────────────────
api.registerTool({
name: "gbrain_query",
description:
"Search the knowledge brain semantically. Returns ranked page excerpts with source paths. " +
"Use for any question about people, companies, deals, meetings, projects, or concepts in the brain.",
parameters: Type.Object({
query: Type.String({ description: "Natural language query" }),
scope: Type.Optional(
Type.Union([
Type.Literal("all"),
Type.Literal("people"),
Type.Literal("companies"),
Type.Literal("deals"),
Type.Literal("meetings"),
Type.Literal("projects"),
Type.Literal("yc"),
Type.Literal("civic"),
], { description: "Limit search to a specific directory/type" })
),
limit: Type.Optional(
Type.Number({ default: 5, minimum: 1, maximum: 20 })
),
includeTimeline: Type.Optional(
Type.Boolean({
default: false,
description:
"Include timeline entries (heavier, use when asking about history)",
})
),
}),
async execute(_id, params) {
const result = await executeQuery(
{
query: params["query"] as string,
scope: params["scope"] as "all" | "people" | "companies" | "deals" | "meetings" | "projects" | "yc" | "civic" | undefined,
limit: params["limit"] as number | undefined,
includeTimeline: params["includeTimeline"] as boolean | undefined,
},
store,
apiKey
);
const lines: string[] = [];
lines.push(
`Found ${result.results.length} results in ${result.queryTimeMs}ms (${result.totalIndexed} pages indexed)\n`
);
for (const r of result.results) {
lines.push(`## ${r.title}`);
lines.push(`Path: ${r.path}`);
lines.push(`Type: ${r.type} | Score: ${r.score} | Updated: ${r.updatedAt}`);
if (r.relatedEntities.length > 0) {
lines.push(`Related: ${r.relatedEntities.join(", ")}`);
}
lines.push(`\n${r.excerpt}\n`);
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_resolve",
description:
"Resolve a name, company, or reference to its brain page path. " +
"Uses exact match, aliases, and embedding similarity. " +
"Returns the page path and compiled truth summary.",
parameters: Type.Object({
name: Type.String({
description:
"Entity name to resolve (e.g. 'Pedro', 'Brex', 'the Variant deal')",
}),
type: Type.Optional(
Type.Union([
Type.Literal("person"),
Type.Literal("company"),
Type.Literal("deal"),
Type.Literal("meeting"),
Type.Literal("any"),
], { default: "any" })
),
}),
async execute(_id, params) {
const result = await executeResolve(
{
name: params["name"] as string,
type: params["type"] as "person" | "company" | "deal" | "meeting" | "any" | undefined,
},
store,
apiKey
);
const lines: string[] = [];
if (result.match) {
const m = result.match;
lines.push(`Resolved: **${m.title}** (confidence: ${m.confidence.toFixed(2)})`);
lines.push(`Path: ${m.path}`);
lines.push(`Type: ${m.type} | Match: ${m.matchReason}`);
if (m.aliases.length > 0) {
lines.push(`Aliases: ${m.aliases.join(", ")}`);
}
lines.push(`\n${m.excerpt}`);
} else {
lines.push(`No confident match found for "${params["name"] as string}".`);
if (result.candidates.length > 0) {
lines.push("\nTop candidates:");
for (const c of result.candidates) {
lines.push(` - ${c.title} (${c.path}) — confidence ${c.confidence.toFixed(2)}, match: ${c.matchReason}`);
}
}
}
lines.push(`\nResolved in ${result.queryTimeMs}ms`);
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_graph",
description:
"Traverse entity relationships in the knowledge brain. " +
"Returns the center entity plus its connected pages with relationship types. " +
"Use to explore who/what a person or company is connected to.",
parameters: Type.Object({
entity: Type.String({
description: "Entity name or path to start from (e.g. 'Pedro', 'companies/brex.md')",
}),
relationship: Type.Optional(
Type.Union([
Type.Literal("mentions"),
Type.Literal("mentioned_by"),
Type.Literal("co_occurs"),
Type.Literal("all"),
], { default: "all", description: "Which direction of edges to follow" })
),
depth: Type.Optional(
Type.Number({ default: 1, minimum: 1, maximum: 3, description: "Traversal depth (1-3)" })
),
}),
async execute(_id, params) {
const result = executeGraph(
{
entity: params["entity"] as string,
relationship: params["relationship"] as "mentions" | "mentioned_by" | "co_occurs" | "all" | undefined,
depth: params["depth"] as number | undefined,
},
store
);
const lines: string[] = [];
if (!result.center) {
lines.push(`No entity found matching "${params["entity"] as string}".`);
} else {
const c = result.center;
lines.push(`## ${c.title} (${c.type})`);
lines.push(`Path: ${c.path}`);
lines.push(`\nFound ${result.edges.length} edges in ${result.queryTimeMs}ms\n`);
const byDepth = new Map<number, typeof result.edges>();
for (const e of result.edges) {
const arr = byDepth.get(e.depth) ?? [];
arr.push(e);
byDepth.set(e.depth, arr);
}
for (const [depth, edges] of [...byDepth.entries()].sort((a, b) => a[0] - b[0])) {
lines.push(`### Depth ${depth}`);
for (const e of edges) {
lines.push(`- **${e.title}** (${e.type}) [${e.relationship}] — ${e.path}`);
if (e.context) lines.push(` > ${e.context}`);
}
}
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_timeline",
description:
"Query temporal changes across the brain. Shows which pages changed when and why. " +
"Supports relative dates like '7d', '30d', 'this week', or ISO dates.",
parameters: Type.Object({
since: Type.String({
description: "Start date: ISO date, '7d', '30d', or 'this week'",
}),
until: Type.Optional(
Type.String({ description: "End date (ISO date). Defaults to now." })
),
entity: Type.Optional(
Type.String({ description: "Filter to a specific entity and its related pages" })
),
scope: Type.Optional(
Type.String({ description: "Filter by directory prefix (e.g. 'companies', 'people')" })
),
}),
async execute(_id, params) {
const result = executeTimeline(
{
since: params["since"] as string,
until: params["until"] as string | undefined,
entity: params["entity"] as string | undefined,
scope: params["scope"] as string | undefined,
},
store,
config.brainPath
);
const lines: string[] = [];
lines.push(`Found ${result.entries.length} changes (${result.queryTimeMs}ms)\n`);
for (const e of result.entries) {
lines.push(`## ${e.title} [${e.changeType}]`);
lines.push(`Path: ${e.path} | Type: ${e.type}`);
lines.push(`Date: ${e.date}`);
lines.push(`Commit: ${e.commitMessage}`);
if (e.timelineExcerpt) {
lines.push(`\n${e.timelineExcerpt}`);
}
lines.push("");
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_ingest",
description:
"Create or update a brain page with automatic re-indexing. " +
"Can create new pages, prepend timeline entries, or replace compiled truth sections.",
parameters: Type.Object({
path: Type.String({
description: "Brain-relative path (e.g. 'people/new-person.md')",
}),
content: Type.Optional(
Type.String({ description: "Full page content for new pages" })
),
timelineEntry: Type.Optional(
Type.String({ description: "Text to prepend as a new timeline entry (date is added automatically)" })
),
compiledTruthUpdate: Type.Optional(
Type.String({ description: "New compiled truth body (replaces everything above the --- separator)" })
),
}),
async execute(_id, params) {
const result = await executeIngest(
{
path: params["path"] as string,
content: params["content"] as string | undefined,
timelineEntry: params["timelineEntry"] as string | undefined,
compiledTruthUpdate: params["compiledTruthUpdate"] as string | undefined,
},
store,
config,
apiKey
);
const lines: string[] = [];
lines.push(`Action: ${result.action}`);
lines.push(`Path: ${result.path}`);
lines.push(`Index: ${result.indexStatus}`);
if (result.errorMessage) {
lines.push(`Error: ${result.errorMessage}`);
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_contradictions",
description:
"Scan brain pages for contradictory numeric facts (e.g. conflicting ARR, headcount, funding figures). " +
"Compares compiled truth against timeline entries to surface disagreements.",
parameters: Type.Object({
entity: Type.Optional(
Type.String({ description: "Check a specific entity only" })
),
scope: Type.Optional(
Type.String({ description: "Check a specific directory (e.g. 'companies')" })
),
limit: Type.Optional(
Type.Number({ default: 10, minimum: 1, maximum: 50 })
),
}),
async execute(_id, params) {
const result = executeContradictions(
{
entity: params["entity"] as string | undefined,
scope: params["scope"] as string | undefined,
limit: params["limit"] as number | undefined,
},
store
);
const lines: string[] = [];
lines.push(
`Found ${result.contradictions.length} contradiction(s) across ${result.pagesChecked} pages (${result.queryTimeMs}ms)\n`
);
for (const c of result.contradictions) {
lines.push(`## ${c.pagePath}${c.field} [${c.severity}]`);
lines.push(` "${c.value1}" from ${c.source1}`);
lines.push(` "${c.value2}" from ${c.source2}`);
lines.push("");
}
if (result.contradictions.length === 0) {
lines.push("No contradictions detected.");
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
api.registerTool({
name: "gbrain_confidence",
description:
"Score the confidence of factual claims in a brain page's compiled truth section. " +
"Checks corroboration count, source recency, and agreement across timeline entries.",
parameters: Type.Object({
entity: Type.String({
description: "Entity to score (e.g. 'Brex', 'Pedro')",
}),
}),
async execute(_id, params) {
const result = executeConfidence(
{ entity: params["entity"] as string },
store
);
const lines: string[] = [];
if (!result.path) {
lines.push(`No entity found matching "${params["entity"] as string}".`);
} else {
lines.push(`## ${result.entity}`);
lines.push(`Path: ${result.path}`);
lines.push(`\nScored ${result.claims.length} factual claims (${result.queryTimeMs}ms)\n`);
for (const claim of result.claims) {
const icon = claim.confidence === "high" ? "✓" : claim.confidence === "medium" ? "~" : "?";
lines.push(`${icon} [${claim.confidence.toUpperCase()}] ${claim.claim}`);
lines.push(` Sources: ${claim.sources.join(", ")}`);
lines.push(` Note: ${claim.note}`);
if (claim.lastVerified) lines.push(` Last verified: ${claim.lastVerified}`);
lines.push("");
}
}
return { content: [{ type: "text", text: lines.join("\n") }] };
},
});
// ── CLI ───────────────────────────────────────────────────────────────────
api.registerCli(registerCli(store, config));
// ── Background Service ────────────────────────────────────────────────────
api.registerService(createWatcherService());
});
+167
View File
@@ -0,0 +1,167 @@
import type { ParsedPage } from "./parser.js";
export type ChunkType = "summary" | "compiled_truth" | "timeline";
export interface Chunk {
chunkType: ChunkType;
content: string;
/** Ordering position within the page */
position: number;
/** Rough token count estimate (chars / 4) */
tokenCount: number;
}
/** Rough token estimator — 1 token ≈ 4 characters */
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
/**
* Build a structured text header from frontmatter fields for the summary chunk.
* This ensures the embedding captures entity metadata (type, title, aliases, tags).
*/
function buildFrontmatterText(page: ParsedPage): string {
const lines: string[] = [];
lines.push(`Title: ${page.title}`);
lines.push(`Type: ${page.type}`);
if (page.aliases.length > 0) {
lines.push(`Aliases: ${page.aliases.join(", ")}`);
}
const tags = page.frontmatter.tags;
if (Array.isArray(tags) && tags.length > 0) {
lines.push(`Tags: ${tags.join(", ")}`);
}
if (page.frontmatter.created) {
lines.push(`Created: ${page.frontmatter.created}`);
}
if (page.frontmatter.updated) {
lines.push(`Updated: ${page.frontmatter.updated}`);
}
return lines.join("\n");
}
/**
* Extract the executive summary — the first paragraph of the compiled truth
* before any section headers appear.
*/
function extractExecutiveSummary(compiledTruth: string): string {
const lines = compiledTruth.split("\n");
const summaryLines: string[] = [];
for (const line of lines) {
if (line.startsWith("#") && summaryLines.length > 0) break;
summaryLines.push(line);
}
return summaryLines.join("\n").trim();
}
/**
* Split timeline section into individual entries.
* Timeline entries typically start with:
* - **YYYY-MM-DD**: ...
* - **YYYY-MM-DD** ...
* ### YYYY-MM-DD
*/
function splitTimelineEntries(timeline: string, maxTokens: number): string[] {
if (!timeline.trim()) return [];
// Split on lines that look like dated entries
const entryBoundary = /^(?:- \*\*\d{4}|\*\*\d{4}|### \d{4}|## \d{4})/m;
const parts = timeline.split(/\n(?=- \*\*\d{4}|\*\*\d{4}|### \d{4}|## \d{4})/);
if (parts.length <= 1) {
// Can't split by date entries — chunk by token count
return chunkByTokens(timeline, maxTokens);
}
// Group small consecutive entries into one chunk
const chunks: string[] = [];
let current = "";
for (const part of parts) {
const combined = current ? `${current}\n${part}` : part;
if (estimateTokens(combined) > maxTokens && current) {
chunks.push(current.trim());
current = part;
} else {
current = combined;
}
}
if (current.trim()) chunks.push(current.trim());
return chunks;
}
/** Split a large text block into chunks of at most maxTokens, breaking at paragraph boundaries. */
function chunkByTokens(text: string, maxTokens: number): string[] {
const paragraphs = text.split(/\n\n+/);
const chunks: string[] = [];
let current = "";
for (const para of paragraphs) {
const combined = current ? `${current}\n\n${para}` : para;
if (estimateTokens(combined) > maxTokens && current) {
chunks.push(current.trim());
current = para;
} else {
current = combined;
}
}
if (current.trim()) chunks.push(current.trim());
return chunks;
}
export function chunkPage(
page: ParsedPage,
maxTokens: number,
indexTimeline: boolean
): Chunk[] {
const chunks: Chunk[] = [];
let position = 0;
// Chunk 1: Frontmatter metadata + executive summary (highest signal)
const frontmatterText = buildFrontmatterText(page);
const executiveSummary = extractExecutiveSummary(page.compiledTruth);
const summaryContent = executiveSummary
? `${frontmatterText}\n\n${executiveSummary}`
: frontmatterText;
if (summaryContent.trim()) {
chunks.push({
chunkType: "summary",
content: summaryContent.trim(),
position: position++,
tokenCount: estimateTokens(summaryContent),
});
}
// Chunk 2+: Compiled truth body (State, Open Threads, See Also, etc.)
const compiledTruthBody = page.compiledTruth.trim();
if (compiledTruthBody) {
const compiledChunks = chunkByTokens(compiledTruthBody, maxTokens);
for (const chunk of compiledChunks) {
if (chunk.trim()) {
chunks.push({
chunkType: "compiled_truth",
content: chunk,
position: position++,
tokenCount: estimateTokens(chunk),
});
}
}
}
// Chunk 3+: Timeline entries (optional)
if (indexTimeline && page.timeline.trim()) {
const timelineChunks = splitTimelineEntries(page.timeline, maxTokens);
for (const chunk of timelineChunks) {
if (chunk.trim()) {
chunks.push({
chunkType: "timeline",
content: chunk,
position: position++,
tokenCount: estimateTokens(chunk),
});
}
}
}
return chunks;
}
+88
View File
@@ -0,0 +1,88 @@
const VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
const VOYAGE_MODEL = "voyage-3";
const VOYAGE_DIMENSIONS = 1024;
const BATCH_SIZE = 100;
export interface EmbedderOptions {
apiKey: string;
model?: string;
}
interface VoyageEmbeddingResponse {
data: Array<{
embedding: number[];
index: number;
}>;
model: string;
usage: {
total_tokens: number;
};
}
async function callVoyageApi(
texts: string[],
apiKey: string,
model: string
): Promise<number[][]> {
const response = await fetch(VOYAGE_API_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
input: texts,
model,
}),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`Voyage API error ${response.status}: ${errorText}`
);
}
const result = (await response.json()) as VoyageEmbeddingResponse;
// Sort by index to ensure correct ordering
const sorted = result.data.sort((a, b) => a.index - b.index);
return sorted.map((d) => d.embedding);
}
/**
* Embed a batch of texts using the Voyage API.
* Automatically batches requests to stay within the 100-text-per-request limit.
*/
export async function embedTexts(
texts: string[],
options: EmbedderOptions
): Promise<number[][]> {
if (texts.length === 0) return [];
const model = options.model && options.model !== "auto"
? options.model
: VOYAGE_MODEL;
const results: number[][] = [];
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
const batch = texts.slice(i, i + BATCH_SIZE);
const embeddings = await callVoyageApi(batch, options.apiKey, model);
results.push(...embeddings);
}
return results;
}
/**
* Embed a single query string.
*/
export async function embedQuery(
query: string,
options: EmbedderOptions
): Promise<number[]> {
const [embedding] = await embedTexts([query], options);
if (!embedding) throw new Error("No embedding returned for query");
return embedding;
}
export { VOYAGE_DIMENSIONS };
+147
View File
@@ -0,0 +1,147 @@
import { readFileSync } from "fs";
import { createHash } from "crypto";
import { relative } from "path";
import matter from "gray-matter";
export interface PageFrontmatter {
title?: string;
type?: string;
created?: string;
updated?: string;
tags?: string[];
aliases?: string[];
sources?: string[];
related?: string[];
attendees?: string[];
investors?: string[];
company?: string;
[key: string]: unknown;
}
export interface ParsedPage {
/** Path relative to brainRoot, e.g. "people/pedro-franceschi.md" */
relativePath: string;
/** Absolute filesystem path */
fullPath: string;
/** SHA-256 of file content */
contentHash: string;
frontmatter: PageFrontmatter;
/** Content above the timeline separator (curated intelligence) */
compiledTruth: string;
/** Content below the timeline separator (chronological entries) */
timeline: string;
/** Related entity paths extracted from wiki-links and markdown links */
relatedPaths: string[];
/** Inferred type (from frontmatter or directory heuristic) */
type: string;
/** Display title (from frontmatter or filename) */
title: string;
/** Aliases for fuzzy entity resolution */
aliases: string[];
}
/** Patterns that signal the start of the timeline section */
const TIMELINE_SEPARATORS = [
/^---$/m,
/^## Timeline$/im,
/^## \d{4}/m,
];
/** Regex to match wiki-links: [[path]] or [[path|label]] */
const WIKI_LINK_RE = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
/** Regex to match markdown links pointing to .md files: [label](path.md) */
const MD_LINK_RE = /\[([^\]]+)\]\(([^)]+\.md)\)/g;
export function parseMarkdown(fullPath: string, brainRoot: string): ParsedPage {
const raw = readFileSync(fullPath, "utf-8");
const contentHash = createHash("sha256").update(raw).digest("hex");
const relativePath = relative(brainRoot, fullPath).replace(/\\/g, "/");
const parsed = matter(raw);
const frontmatter = parsed.data as PageFrontmatter;
const body = parsed.content;
const { compiledTruth, timeline } = splitCompiledTruthAndTimeline(body);
const relatedPaths = extractRelatedPaths(body);
const type = inferType(frontmatter, relativePath);
const title = frontmatter.title ?? inferTitle(relativePath);
const aliases = Array.isArray(frontmatter.aliases) ? frontmatter.aliases : [];
return {
relativePath,
fullPath,
contentHash,
frontmatter,
compiledTruth: compiledTruth.trim(),
timeline: timeline.trim(),
relatedPaths,
type,
title,
aliases,
};
}
function splitCompiledTruthAndTimeline(body: string): {
compiledTruth: string;
timeline: string;
} {
for (const pattern of TIMELINE_SEPARATORS) {
const match = pattern.exec(body);
if (match && match.index !== undefined) {
return {
compiledTruth: body.slice(0, match.index),
timeline: body.slice(match.index + match[0].length),
};
}
}
// No separator found — treat whole body as compiled truth
return { compiledTruth: body, timeline: "" };
}
function extractRelatedPaths(body: string): string[] {
const paths = new Set<string>();
let m: RegExpExecArray | null;
WIKI_LINK_RE.lastIndex = 0;
while ((m = WIKI_LINK_RE.exec(body)) !== null) {
const link = m[1].trim();
// wiki-links may or may not have .md — normalize
paths.add(link.endsWith(".md") ? link : `${link}.md`);
}
MD_LINK_RE.lastIndex = 0;
while ((m = MD_LINK_RE.exec(body)) !== null) {
paths.add(m[2].trim());
}
return Array.from(paths);
}
function inferType(fm: PageFrontmatter, relativePath: string): string {
if (fm.type) return fm.type;
const dir = relativePath.split("/")[0];
const dirTypeMap: Record<string, string> = {
people: "person",
companies: "company",
deals: "deal",
meetings: "meeting",
projects: "project",
yc: "yc",
civic: "civic",
concepts: "concept",
sources: "source",
media: "media",
};
return dirTypeMap[dir] ?? "unknown";
}
function inferTitle(relativePath: string): string {
const filename = relativePath.split("/").pop() ?? relativePath;
return filename
.replace(/\.md$/, "")
.replace(/-/g, " ")
.replace(/\b\w/g, (c) => c.toUpperCase());
}
+483
View File
@@ -0,0 +1,483 @@
import Database from "better-sqlite3";
import { mkdirSync, existsSync } from "fs";
import { dirname } from "path";
import type { ParsedPage } from "./parser.js";
import type { Chunk } from "./chunker.js";
export interface PageRow {
id: number;
path: string;
type: string;
title: string;
aliases: string;
content_hash: string;
frontmatter: string;
compiled_truth: string;
updated_at: string;
indexed_at: string;
}
export interface ChunkRow {
id: number;
pageId: number;
chunkType: string;
content: string;
embedding: Buffer | null;
tokenCount: number;
position: number;
}
export interface ChunkSearchResult {
chunkId: number;
pageId: number;
path: string;
type: string;
title: string;
aliases: string[];
chunkType: string;
content: string;
tokenCount: number;
score: number;
updatedAt: string;
frontmatter: Record<string, unknown>;
}
export interface GraphEdgeRow {
relationship: string;
context: string;
path: string;
title: string;
type: string;
neighborPageId: number;
}
export interface StoreStats {
pageCount: number;
chunkCount: number;
edgeCount: number;
indexSizeBytes: number;
lastSync: string | null;
embeddingModel: string | null;
}
function embeddingToBuffer(embedding: number[]): Buffer {
const arr = new Float64Array(embedding);
return Buffer.from(arr.buffer);
}
function bufferToEmbedding(buf: Buffer): Float64Array {
return new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8);
}
function cosineSimilarity(a: Float64Array, b: Float64Array): number {
let dot = 0;
let normA = 0;
let normB = 0;
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const denom = Math.sqrt(normA) * Math.sqrt(normB);
return denom === 0 ? 0 : dot / denom;
}
export class GBrainStore {
private db: Database.Database;
constructor(indexPath: string) {
const dir = dirname(indexPath);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
this.db = new Database(indexPath);
this.db.pragma("journal_mode = WAL");
this.db.pragma("foreign_keys = ON");
this.initSchema();
}
private initSchema(): void {
this.db.exec(`
CREATE TABLE IF NOT EXISTS pages (
id INTEGER PRIMARY KEY,
path TEXT UNIQUE NOT NULL,
type TEXT NOT NULL DEFAULT 'unknown',
title TEXT NOT NULL DEFAULT '',
aliases TEXT NOT NULL DEFAULT '[]',
content_hash TEXT NOT NULL,
frontmatter TEXT NOT NULL DEFAULT '{}',
compiled_truth TEXT NOT NULL DEFAULT '',
updated_at TEXT NOT NULL DEFAULT '',
indexed_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_pages_path ON pages(path);
CREATE INDEX IF NOT EXISTS idx_pages_type ON pages(type);
CREATE INDEX IF NOT EXISTS idx_pages_title ON pages(title);
CREATE TABLE IF NOT EXISTS chunks (
id INTEGER PRIMARY KEY,
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
chunk_type TEXT NOT NULL,
content TEXT NOT NULL,
embedding BLOB,
token_count INTEGER NOT NULL DEFAULT 0,
position INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_chunks_page_id ON chunks(page_id);
CREATE TABLE IF NOT EXISTS edges (
id INTEGER PRIMARY KEY,
source_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
target_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
relationship TEXT NOT NULL DEFAULT 'mentions',
context TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_page_id);
CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_page_id);
CREATE TABLE IF NOT EXISTS sync_state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
`);
}
upsertPage(page: ParsedPage): number {
const now = new Date().toISOString();
const updatedAt = (page.frontmatter.updated as string) ?? now;
const existing = this.db
.prepare("SELECT id FROM pages WHERE path = ?")
.get(page.relativePath) as { id: number } | undefined;
if (existing) {
this.db
.prepare(
`UPDATE pages SET
type = ?, title = ?, aliases = ?, content_hash = ?,
frontmatter = ?, compiled_truth = ?, updated_at = ?, indexed_at = ?
WHERE path = ?`
)
.run(
page.type,
page.title,
JSON.stringify(page.aliases),
page.contentHash,
JSON.stringify(page.frontmatter),
page.compiledTruth,
updatedAt,
now,
page.relativePath
);
return existing.id;
} else {
const result = this.db
.prepare(
`INSERT INTO pages
(path, type, title, aliases, content_hash, frontmatter, compiled_truth, updated_at, indexed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
page.relativePath,
page.type,
page.title,
JSON.stringify(page.aliases),
page.contentHash,
JSON.stringify(page.frontmatter),
page.compiledTruth,
updatedAt,
now
);
return result.lastInsertRowid as number;
}
}
replaceChunks(pageId: number, chunks: Chunk[], embeddings: number[][]): void {
this.db.prepare("DELETE FROM chunks WHERE page_id = ?").run(pageId);
const insert = this.db.prepare(
`INSERT INTO chunks (page_id, chunk_type, content, embedding, token_count, position)
VALUES (?, ?, ?, ?, ?, ?)`
);
const insertMany = this.db.transaction(
(items: Array<{ chunk: Chunk; embedding: number[] | undefined }>) => {
for (const { chunk, embedding } of items) {
const embBuf = embedding ? embeddingToBuffer(embedding) : null;
insert.run(
pageId,
chunk.chunkType,
chunk.content,
embBuf,
chunk.tokenCount,
chunk.position
);
}
}
);
insertMany(chunks.map((chunk, i) => ({ chunk, embedding: embeddings[i] })));
}
getPageByPath(path: string): PageRow | undefined {
return this.db
.prepare("SELECT * FROM pages WHERE path = ?")
.get(path) as PageRow | undefined;
}
getPageById(id: number): PageRow | undefined {
return this.db
.prepare("SELECT * FROM pages WHERE id = ?")
.get(id) as PageRow | undefined;
}
getAllPages(): PageRow[] {
return this.db.prepare("SELECT * FROM pages").all() as PageRow[];
}
deletePageByPath(path: string): void {
this.db.prepare("DELETE FROM pages WHERE path = ?").run(path);
}
getContentHash(path: string): string | null {
const row = this.db
.prepare("SELECT content_hash FROM pages WHERE path = ?")
.get(path) as { content_hash: string } | undefined;
return row ? row.content_hash : null;
}
/**
* Brute-force cosine similarity search across all chunks that have embeddings.
* Optionally filter by directory scope (path prefix).
*/
searchByEmbedding(
queryEmbedding: number[],
opts: { scope?: string; limit?: number; excludeTimeline?: boolean }
): ChunkSearchResult[] {
const { scope, limit = 5, excludeTimeline = true } = opts;
const queryVec = new Float64Array(queryEmbedding);
let sql = `
SELECT c.id as chunk_id, c.page_id, c.chunk_type, c.content, c.embedding, c.token_count,
p.path, p.type, p.title, p.aliases, p.updated_at, p.frontmatter
FROM chunks c
JOIN pages p ON c.page_id = p.id
WHERE c.embedding IS NOT NULL
`;
const params: unknown[] = [];
if (excludeTimeline) {
sql += " AND c.chunk_type != 'timeline'";
}
if (scope && scope !== "all") {
sql += " AND p.path LIKE ?";
params.push(`${scope}/%`);
}
const rows = this.db.prepare(sql).all(...params) as Array<{
chunk_id: number;
page_id: number;
chunk_type: string;
content: string;
embedding: Buffer;
token_count: number;
path: string;
type: string;
title: string;
aliases: string;
updated_at: string;
frontmatter: string;
}>;
const scored = rows.map((row) => {
const vec = bufferToEmbedding(row.embedding);
const score = cosineSimilarity(queryVec, vec);
return {
chunkId: row.chunk_id,
pageId: row.page_id,
path: row.path,
type: row.type,
title: row.title,
aliases: JSON.parse(row.aliases) as string[],
chunkType: row.chunk_type,
content: row.content,
tokenCount: row.token_count,
score,
updatedAt: row.updated_at,
frontmatter: JSON.parse(row.frontmatter) as Record<string, unknown>,
};
});
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, limit);
}
/**
* Search pages by exact or fuzzy title/alias match.
* Returns pages sorted by match quality.
*/
searchByName(
name: string,
type?: string
): Array<{ page: PageRow; matchType: "exact_path" | "exact_title" | "alias" | "fuzzy" }> {
const normalizedName = name.toLowerCase().trim();
const results: Array<{ page: PageRow; matchType: "exact_path" | "exact_title" | "alias" | "fuzzy" }> = [];
// Exact path match (e.g. "pedro-franceschi" → "people/pedro-franceschi.md")
const slugName = normalizedName.replace(/\s+/g, "-");
const allPages = type && type !== "any"
? (this.db.prepare("SELECT * FROM pages WHERE type = ?").all(type) as PageRow[])
: this.getAllPages();
for (const page of allPages) {
const filename = page.path.split("/").pop()?.replace(/\.md$/, "") ?? "";
const titleLower = page.title.toLowerCase();
const aliases: string[] = JSON.parse(page.aliases);
if (filename === slugName || filename === normalizedName) {
results.push({ page, matchType: "exact_path" });
continue;
}
if (titleLower === normalizedName) {
results.push({ page, matchType: "exact_title" });
continue;
}
if (aliases.some((a) => a.toLowerCase() === normalizedName)) {
results.push({ page, matchType: "alias" });
continue;
}
// Fuzzy: title or alias contains the name
if (
titleLower.includes(normalizedName) ||
aliases.some((a) => a.toLowerCase().includes(normalizedName))
) {
results.push({ page, matchType: "fuzzy" });
}
}
// Sort: exact_path > exact_title > alias > fuzzy
const order = { exact_path: 0, exact_title: 1, alias: 2, fuzzy: 3 };
results.sort((a, b) => order[a.matchType] - order[b.matchType]);
return results;
}
upsertEdges(sourcePageId: number, targetPaths: string[]): void {
const deleteExisting = this.db.prepare(
"DELETE FROM edges WHERE source_page_id = ?"
);
const insert = this.db.prepare(
`INSERT OR IGNORE INTO edges (source_page_id, target_page_id, relationship, context)
SELECT ?, id, 'mentions', '' FROM pages WHERE path = ?`
);
const tx = this.db.transaction(() => {
deleteExisting.run(sourcePageId);
for (const targetPath of targetPaths) {
insert.run(sourcePageId, targetPath);
}
});
tx();
}
/** Get all edges where this page is the source (pages this page mentions). */
getEdgesFrom(pageId: number): GraphEdgeRow[] {
return this.db
.prepare(
`SELECT e.relationship, e.context, p.path, p.title, p.type, p.id as neighborPageId
FROM edges e
JOIN pages p ON e.target_page_id = p.id
WHERE e.source_page_id = ?`
)
.all(pageId) as GraphEdgeRow[];
}
/** Get all edges where this page is the target (pages that mention this page). */
getEdgesTo(pageId: number): GraphEdgeRow[] {
return this.db
.prepare(
`SELECT e.relationship, e.context, p.path, p.title, p.type, p.id as neighborPageId
FROM edges e
JOIN pages p ON e.source_page_id = p.id
WHERE e.target_page_id = ?`
)
.all(pageId) as GraphEdgeRow[];
}
/** Get pages that share edge targets with this page (co-occurrence). */
getCoOccurs(pageId: number): GraphEdgeRow[] {
return this.db
.prepare(
`SELECT DISTINCT 'co_occurs' as relationship, '' as context,
p.path, p.title, p.type, p.id as neighborPageId
FROM edges e1
JOIN edges e2 ON e1.target_page_id = e2.target_page_id
JOIN pages p ON e2.source_page_id = p.id
WHERE e1.source_page_id = ? AND e2.source_page_id != ?`
)
.all(pageId, pageId) as GraphEdgeRow[];
}
/** Get all timeline chunks for a page. */
getTimelineChunks(pageId: number): Array<{ content: string }> {
return this.db
.prepare(
`SELECT content FROM chunks WHERE page_id = ? AND chunk_type = 'timeline'`
)
.all(pageId) as Array<{ content: string }>;
}
getSyncState(key: string): string | null {
const row = this.db
.prepare("SELECT value FROM sync_state WHERE key = ?")
.get(key) as { value: string } | undefined;
return row ? row.value : null;
}
setSyncState(key: string, value: string): void {
this.db
.prepare(
"INSERT OR REPLACE INTO sync_state (key, value) VALUES (?, ?)"
)
.run(key, value);
}
getStats(): StoreStats {
const pageCount = (
this.db.prepare("SELECT COUNT(*) as n FROM pages").get() as { n: number }
).n;
const chunkCount = (
this.db.prepare("SELECT COUNT(*) as n FROM chunks").get() as { n: number }
).n;
const edgeCount = (
this.db.prepare("SELECT COUNT(*) as n FROM edges").get() as { n: number }
).n;
const lastSync = this.getSyncState("last_sync_at");
const embeddingModel = this.getSyncState("embedding_model");
// Approximate: use SQLite page_count pragma
const pageSize = (
this.db.pragma("page_size") as Array<{ page_size: number }>
)[0]?.page_size ?? 4096;
const pageCount2 = (
this.db.pragma("page_count") as Array<{ page_count: number }>
)[0]?.page_count ?? 0;
return {
pageCount,
chunkCount,
edgeCount,
indexSizeBytes: pageSize * pageCount2,
lastSync,
embeddingModel,
};
}
close(): void {
this.db.close();
}
}
+214
View File
@@ -0,0 +1,214 @@
import { execSync } from "child_process";
import { readdirSync, statSync, existsSync } from "fs";
import { join, relative } from "path";
import { parseMarkdown } from "./parser.js";
import { chunkPage } from "./chunker.js";
import { embedTexts } from "./embedder.js";
import { GBrainStore } from "./store.js";
import type { GBrainConfig } from "../types/config.js";
export interface SyncOptions {
dryRun?: boolean;
onProgress?: (indexed: number, total: number, path: string) => void;
}
/** Walk a directory recursively and collect all .md files. */
function collectMarkdownFiles(
dir: string,
excludeDirs: string[]
): string[] {
const files: string[] = [];
function walk(current: string): void {
const entries = readdirSync(current, { withFileTypes: true });
for (const entry of entries) {
const fullPath = join(current, entry.name);
if (entry.isDirectory()) {
if (!excludeDirs.includes(entry.name)) {
walk(fullPath);
}
} else if (entry.isFile() && entry.name.endsWith(".md")) {
files.push(fullPath);
}
}
}
walk(dir);
return files;
}
/** Filter to configured include directories, if any. */
function filterByDirectories(
files: string[],
brainPath: string,
includeDirs: string[]
): string[] {
if (includeDirs.length === 0) return files;
return files.filter((f) => {
const rel = relative(brainPath, f);
return includeDirs.some((d) => rel.startsWith(d + "/") || rel.startsWith(d + "\\"));
});
}
async function indexFiles(
filePaths: string[],
brainPath: string,
store: GBrainStore,
config: GBrainConfig,
opts: SyncOptions
): Promise<void> {
const apiKey = process.env["VOYAGE_API_KEY"] ?? "";
const embedderOpts = { apiKey, model: config.embeddingModel };
const total = filePaths.length;
let indexed = 0;
// Process in batches to avoid holding too many files in memory
const BATCH = 20;
for (let i = 0; i < filePaths.length; i += BATCH) {
const batch = filePaths.slice(i, i + BATCH);
for (const fullPath of batch) {
const relativePath = relative(brainPath, fullPath).replace(/\\/g, "/");
try {
const page = parseMarkdown(fullPath, brainPath);
// Skip unchanged files
const existingHash = store.getContentHash(relativePath);
if (existingHash === page.contentHash && !opts.dryRun) {
indexed++;
opts.onProgress?.(indexed, total, relativePath);
continue;
}
if (opts.dryRun) {
indexed++;
opts.onProgress?.(indexed, total, relativePath);
continue;
}
const chunks = chunkPage(page, config.chunkMaxTokens, config.indexTimeline);
const texts = chunks.map((c) => c.content);
const embeddings = apiKey
? await embedTexts(texts, embedderOpts)
: texts.map(() => [] as number[]);
const pageId = store.upsertPage(page);
store.replaceChunks(pageId, chunks, embeddings);
store.upsertEdges(pageId, page.relatedPaths);
indexed++;
opts.onProgress?.(indexed, total, relativePath);
} catch (err) {
// Log and skip — don't abort the whole sync for one bad file
console.error(`[gbrain] Failed to index ${relativePath}:`, err);
indexed++;
}
}
}
}
/** Full reindex: walk all .md files in brainPath and index them. */
export async function fullReindex(
store: GBrainStore,
config: GBrainConfig,
opts: SyncOptions = {}
): Promise<{ indexed: number; total: number }> {
const allFiles = collectMarkdownFiles(config.brainPath, config.excludeDirectories);
const filtered = filterByDirectories(allFiles, config.brainPath, config.directories);
await indexFiles(filtered, config.brainPath, store, config, opts);
if (!opts.dryRun) {
store.setSyncState("last_sync_at", new Date().toISOString());
store.setSyncState("embedding_model", config.embeddingModel);
// Record current HEAD if in a git repo
try {
const head = execSync("git rev-parse HEAD", {
cwd: config.brainPath,
stdio: ["pipe", "pipe", "pipe"],
})
.toString()
.trim();
store.setSyncState("last_synced_head", head);
} catch {
// Not a git repo — fine
}
}
return { indexed: filtered.length, total: filtered.length };
}
/** Incremental reindex: only re-index files changed between two git commits. */
export async function incrementalReindex(
fromHead: string,
toHead: string,
store: GBrainStore,
config: GBrainConfig,
opts: SyncOptions = {}
): Promise<{ indexed: number; deleted: number }> {
let changedFiles: string[] = [];
let deletedFiles: string[] = [];
try {
const diffOutput = execSync(
`git diff --name-status ${fromHead} ${toHead} -- "*.md"`,
{ cwd: config.brainPath, stdio: ["pipe", "pipe", "pipe"] }
).toString();
for (const line of diffOutput.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
const [status, ...pathParts] = trimmed.split(/\s+/);
const filePath = pathParts.join(" ");
if (!filePath) continue;
if (status === "D") {
deletedFiles.push(filePath);
} else if (status?.match(/^[ACMR]/)) {
changedFiles.push(filePath);
}
}
} catch {
// git diff failed — fall back to full reindex
console.error("[gbrain] git diff failed, falling back to full reindex");
const result = await fullReindex(store, config, opts);
return { indexed: result.indexed, deleted: 0 };
}
// Handle deletions
if (!opts.dryRun) {
for (const filePath of deletedFiles) {
store.deletePageByPath(filePath);
}
}
// Convert relative paths to absolute for indexing
const absolutePaths = changedFiles
.map((f) => join(config.brainPath, f))
.filter((f) => existsSync(f));
await indexFiles(absolutePaths, config.brainPath, store, config, opts);
if (!opts.dryRun) {
store.setSyncState("last_sync_at", new Date().toISOString());
store.setSyncState("last_synced_head", toHead);
}
return { indexed: absolutePaths.length, deleted: deletedFiles.length };
}
/** Get the current git HEAD of the brain repo. */
export function getCurrentHead(brainPath: string): string | null {
try {
return execSync("git rev-parse HEAD", {
cwd: brainPath,
stdio: ["pipe", "pipe", "pipe"],
})
.toString()
.trim();
} catch {
return null;
}
}
+71
View File
@@ -0,0 +1,71 @@
import type { PluginService, ServiceContext } from "openclaw/plugin-sdk/plugin-entry";
import { getCurrentHead, incrementalReindex } from "./indexer/sync.js";
import { GBrainStore } from "./indexer/store.js";
import { resolveConfig } from "./types/config.js";
export function createWatcherService(): PluginService {
let intervalHandle: ReturnType<typeof setInterval> | null = null;
let store: GBrainStore | null = null;
return {
id: "gbrain-watcher",
async start(ctx: ServiceContext): Promise<void> {
const config = resolveConfig(ctx.config);
store = new GBrainStore(config.indexPath);
const pollMs = config.watchInterval * 1000;
let lastHead = store.getSyncState("last_synced_head");
// Sync on startup if we have a recorded head
const currentHead = getCurrentHead(config.brainPath);
if (currentHead && lastHead && currentHead !== lastHead) {
console.log(
`[gbrain-watcher] Detected ${currentHead.slice(0, 8)} vs last synced ${lastHead.slice(0, 8)} — running incremental reindex on start`
);
try {
await incrementalReindex(lastHead, currentHead, store, config);
lastHead = currentHead;
} catch (err) {
console.error("[gbrain-watcher] Startup reindex failed:", err);
}
} else if (currentHead) {
lastHead = currentHead;
}
intervalHandle = setInterval(async () => {
try {
const head = getCurrentHead(config.brainPath);
if (!head) return; // Not a git repo
if (head !== lastHead) {
const from = lastHead ?? head;
console.log(
`[gbrain-watcher] Brain repo changed: ${from.slice(0, 8)}${head.slice(0, 8)}`
);
await incrementalReindex(from, head, store!, config);
lastHead = head;
}
} catch (err) {
console.error("[gbrain-watcher] Poll error:", err);
}
}, pollMs);
console.log(
`[gbrain-watcher] Started. Polling every ${config.watchInterval}s for changes in ${config.brainPath}`
);
},
async stop(): Promise<void> {
if (intervalHandle !== null) {
clearInterval(intervalHandle);
intervalHandle = null;
}
if (store) {
store.close();
store = null;
}
console.log("[gbrain-watcher] Stopped.");
},
};
}
+176
View File
@@ -0,0 +1,176 @@
import type { GBrainStore } from "../indexer/store.js";
export interface ConfidenceParams {
entity: string;
}
export interface ClaimConfidence {
claim: string;
confidence: "high" | "medium" | "low";
sources: string[];
lastVerified: string | null;
note: string;
}
export interface ConfidenceResult {
entity: string;
path: string | null;
claims: ClaimConfidence[];
queryTimeMs: number;
}
/** Patterns that signal a sentence contains a concrete factual claim. */
const FACTUAL_PATTERNS: RegExp[] = [
/\$[\d,.]+\s*(M|B|K|million|billion|thousand)?/i,
/\b\d{4}-\d{2}-\d{2}\b/,
/\b\d+\s*(employees|people|users|customers)\b/i,
/\b(Series [A-Z]|seed round|pre-seed)\b/i,
/\b(raised|founded|acquired|valued|funded|launched)\b.*\b\d/i,
];
function isFactualClaim(sentence: string): boolean {
return FACTUAL_PATTERNS.some((p) => p.test(sentence));
}
function splitSentences(text: string): string[] {
return text
.split(/[.\n]+/)
.map((s) => s.trim())
.filter((s) => s.length > 15);
}
/**
* Extract key numeric/date terms from a claim to look for in the timeline.
*/
function extractKeyTerms(claim: string): string[] {
const terms: string[] = [];
const numMatches = claim.match(/\$?[\d,.]+\s*(M|B|K|million|billion|%)?/gi);
if (numMatches) terms.push(...numMatches.map((t) => t.trim()).filter((t) => t.length > 1));
const dateMatches = claim.match(/\b\d{4}(?:-\d{2}-\d{2})?\b/g);
if (dateMatches) terms.push(...dateMatches);
return terms;
}
/**
* Count how many timeline lines corroborate the claim by sharing key terms.
*/
function countCorroborations(claim: string, timelineText: string): number {
if (!timelineText) return 0;
const keyTerms = extractKeyTerms(claim);
if (keyTerms.length === 0) return 0;
const threshold = Math.ceil(keyTerms.length / 2);
let count = 0;
for (const line of timelineText.split("\n")) {
const lineUpper = line.toUpperCase();
const matches = keyTerms.filter((t) => lineUpper.includes(t.toUpperCase()));
if (matches.length >= threshold) count++;
}
return count;
}
/** Find the most recent ISO date in the timeline text. */
function mostRecentDate(timelineText: string): string | null {
const dates = timelineText.match(/\b\d{4}-\d{2}-\d{2}\b/g);
if (!dates || dates.length === 0) return null;
return dates.slice().sort().reverse()[0] ?? null;
}
function scoreConfidence(
corroborations: number,
lastVerified: string | null
): { confidence: "high" | "medium" | "low"; note: string } {
const ageMonths = lastVerified
? (Date.now() - new Date(lastVerified).getTime()) / (1000 * 60 * 60 * 24 * 30)
: Infinity;
if (corroborations >= 3 && ageMonths < 6) {
return {
confidence: "high",
note: `${corroborations} sources, verified ${Math.round(ageMonths)}mo ago`,
};
}
if (corroborations >= 1 && ageMonths < 12) {
return {
confidence: "medium",
note: `${corroborations} source(s), verified ${Math.round(ageMonths)}mo ago`,
};
}
if (ageMonths > 18) {
return {
confidence: "low",
note: `Last verified ${Math.round(ageMonths)}mo ago — stale`,
};
}
if (corroborations >= 3) {
return { confidence: "high", note: `${corroborations} corroborating sources` };
}
if (corroborations >= 1) {
return { confidence: "medium", note: `${corroborations} source(s)` };
}
return { confidence: "low", note: "Single source or uncorroborated" };
}
export function executeConfidence(
params: ConfidenceParams,
store: GBrainStore
): ConfidenceResult {
const startMs = Date.now();
const { entity } = params;
const matches = store.searchByName(entity);
if (matches.length === 0) {
return { entity, path: null, claims: [], queryTimeMs: Date.now() - startMs };
}
const match = matches[0];
if (!match) {
return { entity, path: null, claims: [], queryTimeMs: Date.now() - startMs };
}
const page = match.page;
// Get timeline content from indexed chunks
const timelineChunks = store.getTimelineChunks(page.id);
const timelineText = timelineChunks.map((c) => c.content).join("\n");
const lastVerified = mostRecentDate(timelineText);
// Parse frontmatter sources for attribution
const frontmatter = JSON.parse(page.frontmatter) as Record<string, unknown>;
const fmSources = Array.isArray(frontmatter["sources"])
? (frontmatter["sources"] as string[])
: [];
const factualSentences = splitSentences(page.compiled_truth).filter(isFactualClaim);
const claims: ClaimConfidence[] = factualSentences.slice(0, 20).map((sentence) => {
const corroborations = countCorroborations(sentence, timelineText);
const { confidence, note } = scoreConfidence(corroborations, lastVerified);
const sources: string[] =
corroborations > 0 && fmSources.length > 0
? fmSources.slice(0, Math.min(corroborations, fmSources.length))
: ["compiled_truth"];
return {
claim: sentence,
confidence,
sources,
lastVerified: corroborations > 0 ? lastVerified : null,
note,
};
});
return {
entity: page.title,
path: page.path,
claims,
queryTimeMs: Date.now() - startMs,
};
}
+172
View File
@@ -0,0 +1,172 @@
import type { GBrainStore, PageRow } from "../indexer/store.js";
export interface ContradictionsParams {
entity?: string;
scope?: string;
limit?: number;
}
export interface Contradiction {
pagePath: string;
field: string;
value1: string;
source1: string;
value2: string;
source2: string;
severity: "high" | "medium" | "low";
}
export interface ContradictionsResult {
contradictions: Contradiction[];
pagesChecked: number;
queryTimeMs: number;
}
interface NumericFact {
value: string;
normalizedNumber: number;
metric: string;
source: string;
date?: string;
}
const METRIC_KEYWORDS = [
"ARR", "MRR", "revenue", "funding", "valuation",
"headcount", "employees", "raised", "round", "users", "customers",
];
// Matches currency/numeric patterns: $1.2M, 500K, 2.3 million, 45%, etc.
const NUMERIC_RE = /\$?(\d[\d,.]*)(\s*(?:M|B|K|million|billion|thousand|%))?/gi;
function extractNumericFacts(text: string, source: string): NumericFact[] {
const facts: NumericFact[] = [];
for (const line of text.split("\n")) {
const lineUpper = line.toUpperCase();
const metric = METRIC_KEYWORDS.find((m) => lineUpper.includes(m.toUpperCase()));
if (!metric) continue;
const dateMatch = line.match(/\d{4}-\d{2}-\d{2}/);
const date = dateMatch ? dateMatch[0] : undefined;
NUMERIC_RE.lastIndex = 0;
let m: RegExpExecArray | null;
while ((m = NUMERIC_RE.exec(line)) !== null) {
const rawStr = m[1].replace(/,/g, "");
const rawNumber = parseFloat(rawStr);
if (isNaN(rawNumber) || rawNumber === 0) continue;
const unit = (m[2] ?? "").trim().toUpperCase();
let normalizedNumber = rawNumber;
if (unit === "M" || unit === "MILLION") normalizedNumber = rawNumber * 1_000_000;
else if (unit === "B" || unit === "BILLION") normalizedNumber = rawNumber * 1_000_000_000;
else if (unit === "K" || unit === "THOUSAND") normalizedNumber = rawNumber * 1_000;
facts.push({
value: m[0].trim(),
normalizedNumber,
metric,
source,
date,
});
}
}
return facts;
}
function detectPageContradictions(
page: PageRow,
timelineText: string
): Contradiction[] {
const contradictions: Contradiction[] = [];
const allFacts: NumericFact[] = [
...extractNumericFacts(page.compiled_truth, "compiled_truth"),
...extractNumericFacts(timelineText, "timeline"),
];
// Group by metric
const byMetric = new Map<string, NumericFact[]>();
for (const fact of allFacts) {
const group = byMetric.get(fact.metric) ?? [];
group.push(fact);
byMetric.set(fact.metric, group);
}
for (const [metric, facts] of byMetric) {
if (facts.length < 2) continue;
for (let i = 0; i < facts.length - 1; i++) {
for (let j = i + 1; j < facts.length; j++) {
const f1 = facts[i];
const f2 = facts[j];
if (!f1 || !f2) continue;
const maxVal = Math.max(f1.normalizedNumber, f2.normalizedNumber);
if (maxVal === 0) continue;
const diff = Math.abs(f1.normalizedNumber - f2.normalizedNumber) / maxVal;
if (diff <= 0.1) continue; // within 10% — not a contradiction
const severity: Contradiction["severity"] =
diff > 0.5 ? "high" : diff > 0.2 ? "medium" : "low";
contradictions.push({
pagePath: page.path,
field: metric,
value1: f1.value,
source1: f1.source + (f1.date ? ` (${f1.date})` : ""),
value2: f2.value,
source2: f2.source + (f2.date ? ` (${f2.date})` : ""),
severity,
});
}
}
}
return contradictions;
}
export function executeContradictions(
params: ContradictionsParams,
store: GBrainStore
): ContradictionsResult {
const startMs = Date.now();
const { entity, scope, limit = 10 } = params;
let pagesToCheck: PageRow[];
if (entity) {
const matches = store.searchByName(entity);
if (matches.length === 0) {
return { contradictions: [], pagesChecked: 0, queryTimeMs: Date.now() - startMs };
}
const match = matches[0];
pagesToCheck = match ? [match.page] : [];
} else if (scope) {
pagesToCheck = store
.getAllPages()
.filter((p) => p.path.startsWith(scope + "/"));
} else {
pagesToCheck = store.getAllPages();
}
const contradictions: Contradiction[] = [];
for (const page of pagesToCheck) {
if (contradictions.length >= limit) break;
// Get timeline content from indexed chunks
const timelineChunks = store.getTimelineChunks(page.id);
const timelineText = timelineChunks.map((c) => c.content).join("\n");
const found = detectPageContradictions(page, timelineText);
contradictions.push(...found);
}
return {
contradictions: contradictions.slice(0, limit),
pagesChecked: pagesToCheck.length,
queryTimeMs: Date.now() - startMs,
};
}
+107
View File
@@ -0,0 +1,107 @@
import type { GBrainStore, GraphEdgeRow } from "../indexer/store.js";
export interface GraphParams {
entity: string;
relationship?: "mentions" | "mentioned_by" | "co_occurs" | "all";
depth?: number;
}
export interface GraphNode {
path: string;
title: string;
type: string;
}
export interface GraphEdge {
path: string;
title: string;
type: string;
relationship: string;
context: string;
depth: number;
}
export interface GraphResult {
center: GraphNode | null;
edges: GraphEdge[];
queryTimeMs: number;
}
function fetchEdgesForPage(
pageId: number,
relationship: "mentions" | "mentioned_by" | "co_occurs" | "all",
store: GBrainStore
): GraphEdgeRow[] {
switch (relationship) {
case "mentions":
return store.getEdgesFrom(pageId);
case "mentioned_by":
return store.getEdgesTo(pageId);
case "co_occurs":
return store.getCoOccurs(pageId);
case "all":
return [
...store.getEdgesFrom(pageId),
...store.getEdgesTo(pageId),
...store.getCoOccurs(pageId),
];
}
}
export function executeGraph(params: GraphParams, store: GBrainStore): GraphResult {
const startMs = Date.now();
const { entity, relationship = "all", depth: maxDepth = 1 } = params;
const clampedDepth = Math.min(Math.max(maxDepth, 1), 3);
// Resolve entity to a page
const textMatches = store.searchByName(entity);
const firstMatch = textMatches[0];
if (!firstMatch) {
return { center: null, edges: [], queryTimeMs: Date.now() - startMs };
}
const centerPage = firstMatch.page;
const center: GraphNode = {
path: centerPage.path,
title: centerPage.title,
type: centerPage.type,
};
const allEdges: GraphEdge[] = [];
// Track visited page IDs and edge keys to avoid duplicates
const visitedPageIds = new Set<number>([centerPage.id]);
const seenEdgeKeys = new Set<string>();
let frontier: number[] = [centerPage.id];
for (let d = 1; d <= clampedDepth; d++) {
const nextFrontier: number[] = [];
for (const pageId of frontier) {
const rawEdges = fetchEdgesForPage(pageId, relationship, store);
for (const raw of rawEdges) {
const edgeKey = `${raw.path}::${raw.relationship}`;
if (seenEdgeKeys.has(edgeKey)) continue;
seenEdgeKeys.add(edgeKey);
allEdges.push({
path: raw.path,
title: raw.title,
type: raw.type,
relationship: raw.relationship,
context: raw.context,
depth: d,
});
if (!visitedPageIds.has(raw.neighborPageId)) {
visitedPageIds.add(raw.neighborPageId);
nextFrontier.push(raw.neighborPageId);
}
}
}
frontier = nextFrontier;
if (frontier.length === 0) break;
}
return { center, edges: allEdges, queryTimeMs: Date.now() - startMs };
}
+184
View File
@@ -0,0 +1,184 @@
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
import { join, dirname } from "path";
import { parseMarkdown } from "../indexer/parser.js";
import { chunkPage } from "../indexer/chunker.js";
import { embedTexts } from "../indexer/embedder.js";
import type { GBrainStore } from "../indexer/store.js";
import type { GBrainConfig } from "../types/config.js";
export interface IngestParams {
path: string;
content?: string;
timelineEntry?: string;
compiledTruthUpdate?: string;
}
export interface IngestResult {
path: string;
action: "created" | "updated_timeline" | "updated_truth" | "created_with_content";
indexStatus: "indexed" | "skipped" | "error";
errorMessage?: string;
}
/**
* Split a raw markdown file into its YAML frontmatter and body.
* The frontmatter is delimited by opening and closing "---" lines.
*/
function splitFrontmatterAndBody(raw: string): { front: string; body: string } {
if (!raw.startsWith("---")) {
return { front: "", body: raw };
}
// Find the newline after the opening ---
const firstNl = raw.indexOf("\n");
if (firstNl === -1) return { front: raw, body: "" };
const afterOpen = raw.slice(firstNl + 1);
const closingMatch = /^---\s*$/m.exec(afterOpen);
if (!closingMatch || closingMatch.index === undefined) {
return { front: raw, body: "" };
}
const closingEnd = firstNl + 1 + closingMatch.index + closingMatch[0].length;
return {
front: raw.slice(0, closingEnd),
body: raw.slice(closingEnd),
};
}
/** Timeline separator patterns (searched in the body, after frontmatter). */
const SEPARATOR_PATTERNS = [
/^---\s*$/m,
/^## Timeline\s*$/im,
/^## \d{4}/m,
];
function findTimelineSeparator(body: string): { index: number; length: number } | null {
for (const pattern of SEPARATOR_PATTERNS) {
const match = pattern.exec(body);
if (match && match.index !== undefined) {
return { index: match.index, length: match[0].length };
}
}
return null;
}
/**
* Prepend a new timeline entry (reverse-chronological) right after the timeline
* separator. If no separator exists, appends a new "---" section.
*/
function prependTimelineEntry(raw: string, entry: string): string {
const today = new Date().toISOString().slice(0, 10);
const formattedEntry = `- **${today}**: ${entry}`;
const { front, body } = splitFrontmatterAndBody(raw);
const sep = findTimelineSeparator(body);
if (!sep) {
// No timeline section — append one
return `${raw.trimEnd()}\n\n---\n\n${formattedEntry}\n`;
}
const sepEnd = sep.index + sep.length;
const beforeTimeline = body.slice(0, sepEnd);
const afterTimeline = body.slice(sepEnd);
return `${front}${beforeTimeline}\n\n${formattedEntry}${afterTimeline}`;
}
/**
* Replace the compiled truth section (everything between frontmatter and the
* timeline separator), preserving frontmatter and timeline.
*/
function replaceCompiledTruth(raw: string, newTruth: string): string {
const { front, body } = splitFrontmatterAndBody(raw);
const sep = findTimelineSeparator(body);
if (!sep) {
// No timeline — replace entire body
return `${front}\n\n${newTruth.trimEnd()}\n`;
}
const timeline = body.slice(sep.index);
return `${front}\n\n${newTruth.trimEnd()}\n\n${timeline}`;
}
export async function executeIngest(
params: IngestParams,
store: GBrainStore,
config: GBrainConfig,
apiKey: string
): Promise<IngestResult> {
const { path: relativePath, content, timelineEntry, compiledTruthUpdate } = params;
const fullPath = join(config.brainPath, relativePath);
let action: IngestResult["action"];
try {
const fileExists = existsSync(fullPath);
if (fileExists && timelineEntry) {
const raw = readFileSync(fullPath, "utf-8");
writeFileSync(fullPath, prependTimelineEntry(raw, timelineEntry), "utf-8");
action = "updated_timeline";
} else if (fileExists && compiledTruthUpdate) {
const raw = readFileSync(fullPath, "utf-8");
writeFileSync(fullPath, replaceCompiledTruth(raw, compiledTruthUpdate), "utf-8");
action = "updated_truth";
} else if (!fileExists && content) {
// Ensure parent directory exists
const dir = dirname(fullPath);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
writeFileSync(fullPath, content, "utf-8");
action = "created_with_content";
} else if (!fileExists) {
// Create minimal template
const dir = dirname(fullPath);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
const title =
relativePath
.split("/")
.pop()
?.replace(/\.md$/, "")
.replace(/-/g, " ") ?? relativePath;
writeFileSync(fullPath, `---\ntitle: ${title}\n---\n\n`, "utf-8");
action = "created";
} else {
// File exists, no mutation specified — nothing to do
return { path: relativePath, action: "updated_truth", indexStatus: "skipped" };
}
} catch (err) {
return {
path: relativePath,
action: "created",
indexStatus: "error",
errorMessage: err instanceof Error ? err.message : String(err),
};
}
// Re-index the written file
try {
const page = parseMarkdown(fullPath, config.brainPath);
const chunks = chunkPage(page, config.chunkMaxTokens, config.indexTimeline);
const texts = chunks.map((c) => c.content);
const embeddings = apiKey
? await embedTexts(texts, { apiKey, model: config.embeddingModel })
: texts.map(() => [] as number[]);
const pageId = store.upsertPage(page);
store.replaceChunks(pageId, chunks, embeddings);
store.upsertEdges(pageId, page.relatedPaths);
return { path: relativePath, action, indexStatus: "indexed" };
} catch (err) {
return {
path: relativePath,
action,
indexStatus: "error",
errorMessage: err instanceof Error ? err.message : String(err),
};
}
}
+110
View File
@@ -0,0 +1,110 @@
import { embedQuery } from "../indexer/embedder.js";
import type { GBrainStore, ChunkSearchResult } from "../indexer/store.js";
export interface QueryParams {
query: string;
scope?: "all" | "people" | "companies" | "deals" | "meetings" | "projects" | "yc" | "civic";
limit?: number;
includeTimeline?: boolean;
}
export interface QueryResultItem {
path: string;
type: string;
title: string;
score: number;
excerpt: string;
updatedAt: string;
relatedEntities: string[];
}
export interface QueryResult {
results: QueryResultItem[];
totalIndexed: number;
queryTimeMs: number;
}
/** Scope value to directory prefix mapping */
const SCOPE_TO_DIR: Record<string, string> = {
people: "people",
companies: "companies",
deals: "deals",
meetings: "meetings",
projects: "projects",
yc: "yc",
civic: "civic",
};
/** Deduplicate results: keep only the best-scoring chunk per page. */
function deduplicateByPage(results: ChunkSearchResult[]): ChunkSearchResult[] {
const best = new Map<number, ChunkSearchResult>();
for (const r of results) {
const existing = best.get(r.pageId);
if (!existing || r.score > existing.score) {
best.set(r.pageId, r);
}
}
return Array.from(best.values()).sort((a, b) => b.score - a.score);
}
function truncateExcerpt(text: string, maxChars = 400): string {
if (text.length <= maxChars) return text;
return text.slice(0, maxChars).replace(/\s\S*$/, "") + "…";
}
export async function executeQuery(
params: QueryParams,
store: GBrainStore,
apiKey: string
): Promise<QueryResult> {
const startMs = Date.now();
const {
query,
scope = "all",
limit = 5,
includeTimeline = false,
} = params;
// Embed the query
const queryEmbedding = await embedQuery(query, { apiKey });
const dirScope = scope !== "all" ? (SCOPE_TO_DIR[scope] ?? scope) : undefined;
// Search: fetch more than needed to allow deduplication
const raw = store.searchByEmbedding(queryEmbedding, {
scope: dirScope,
limit: limit * 4,
excludeTimeline: !includeTimeline,
});
// Deduplicate per page then take top N
const deduped = deduplicateByPage(raw).slice(0, limit);
// For each result, fetch related entities from edges table (via getPageById)
const stats = store.getStats();
const results: QueryResultItem[] = deduped.map((hit) => {
const fm = hit.frontmatter;
const related: string[] = [];
if (Array.isArray(fm["related"])) {
related.push(...(fm["related"] as string[]));
}
return {
path: hit.path,
type: hit.type,
title: hit.title,
score: Math.round(hit.score * 1000) / 1000,
excerpt: truncateExcerpt(hit.content),
updatedAt: hit.updatedAt,
relatedEntities: related,
};
});
return {
results,
totalIndexed: stats.pageCount,
queryTimeMs: Date.now() - startMs,
};
}
+108
View File
@@ -0,0 +1,108 @@
import { embedQuery } from "../indexer/embedder.js";
import type { GBrainStore } from "../indexer/store.js";
export interface ResolveParams {
name: string;
type?: "person" | "company" | "deal" | "meeting" | "any";
}
export interface ResolveCandidate {
path: string;
type: string;
title: string;
confidence: number;
matchReason: string;
excerpt: string;
aliases: string[];
}
export interface ResolveResult {
/** Best match, or null if nothing found */
match: ResolveCandidate | null;
/** Top-3 candidates for disambiguation */
candidates: ResolveCandidate[];
queryTimeMs: number;
}
const MATCH_CONFIDENCE: Record<string, number> = {
exact_path: 0.99,
exact_title: 0.97,
alias: 0.92,
fuzzy: 0.75,
embedding: 0.60,
};
function truncate(text: string, maxChars = 300): string {
if (text.length <= maxChars) return text;
return text.slice(0, maxChars).replace(/\s\S*$/, "") + "…";
}
export async function executeResolve(
params: ResolveParams,
store: GBrainStore,
apiKey: string
): Promise<ResolveResult> {
const startMs = Date.now();
const { name, type = "any" } = params;
const candidates: ResolveCandidate[] = [];
// 14: Exact/fuzzy text match on filename, title, aliases
const textMatches = store.searchByName(name, type);
for (const { page, matchType } of textMatches.slice(0, 10)) {
candidates.push({
path: page.path,
type: page.type,
title: page.title,
confidence: MATCH_CONFIDENCE[matchType] ?? 0.5,
matchReason: matchType,
excerpt: truncate(page.compiled_truth),
aliases: JSON.parse(page.aliases) as string[],
});
}
// 5: Embedding similarity fallback (only if we have few text matches)
if (candidates.length < 3 && apiKey) {
try {
const queryEmbedding = await embedQuery(name, { apiKey });
const embeddingMatches = store.searchByEmbedding(queryEmbedding, {
scope: type !== "any" ? `${type}s` : undefined,
limit: 5,
excludeTimeline: true,
});
for (const hit of embeddingMatches) {
// Don't add duplicates already found by text match
const alreadyFound = candidates.some((c) => c.path === hit.path);
if (!alreadyFound) {
candidates.push({
path: hit.path,
type: hit.type,
title: hit.title,
confidence: Math.round(hit.score * MATCH_CONFIDENCE["embedding"]! * 100) / 100,
matchReason: "embedding",
excerpt: truncate(hit.content),
aliases: hit.aliases,
});
}
}
} catch {
// Embedding lookup failed — ignore, text matches are enough
}
}
// Sort by confidence
candidates.sort((a, b) => b.confidence - a.confidence);
const top3 = candidates.slice(0, 3);
const best = top3[0] ?? null;
// Only return a match if confidence is reasonable
const match = best && best.confidence >= 0.6 ? best : null;
return {
match,
candidates: top3,
queryTimeMs: Date.now() - startMs,
};
}
+213
View File
@@ -0,0 +1,213 @@
import { execSync } from "child_process";
import type { GBrainStore } from "../indexer/store.js";
export interface TimelineParams {
since: string;
until?: string;
entity?: string;
scope?: string;
}
export interface TimelineEntry {
date: string;
path: string;
title: string;
type: string;
changeType: "added" | "modified" | "deleted";
commitMessage: string;
timelineExcerpt: string;
}
export interface TimelineResult {
entries: TimelineEntry[];
queryTimeMs: number;
}
/** Convert relative date strings to ISO date strings. */
function parseRelativeDate(since: string): string {
const now = new Date();
const lower = since.toLowerCase().trim();
if (lower === "this week") {
const day = now.getDay(); // 0=Sunday
const diffToMonday = day === 0 ? -6 : 1 - day;
const monday = new Date(now);
monday.setDate(now.getDate() + diffToMonday);
monday.setHours(0, 0, 0, 0);
return monday.toISOString();
}
const daysMatch = lower.match(/^(\d+)d$/);
if (daysMatch) {
const days = parseInt(daysMatch[1], 10);
const d = new Date(now);
d.setDate(d.getDate() - days);
return d.toISOString();
}
// Assume it's already an ISO date string
return since;
}
interface GitCommit {
hash: string;
date: string;
subject: string;
files: Array<{ path: string; changeType: "added" | "modified" | "deleted" }>;
}
function parseGitLog(
brainPath: string,
sinceDate: string,
untilDate?: string
): GitCommit[] {
let cmd = `git log --since="${sinceDate}"`;
if (untilDate) cmd += ` --until="${untilDate}"`;
// Use a sentinel prefix so we can reliably distinguish commit lines from file lines
cmd += ` --pretty=format:"COMMIT|%H|%ai|%s" --name-status -- "*.md"`;
let output: string;
try {
output = execSync(cmd, {
cwd: brainPath,
stdio: ["pipe", "pipe", "pipe"],
maxBuffer: 10 * 1024 * 1024,
}).toString();
} catch {
return [];
}
const commits: GitCommit[] = [];
let current: GitCommit | null = null;
for (const line of output.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
if (trimmed.startsWith("COMMIT|")) {
if (current) commits.push(current);
const rest = trimmed.slice("COMMIT|".length);
const pipeIdx1 = rest.indexOf("|");
const pipeIdx2 = rest.indexOf("|", pipeIdx1 + 1);
const hash = pipeIdx1 >= 0 ? rest.slice(0, pipeIdx1) : rest;
const date = pipeIdx2 >= 0 ? rest.slice(pipeIdx1 + 1, pipeIdx2) : "";
const subject = pipeIdx2 >= 0 ? rest.slice(pipeIdx2 + 1) : "";
current = { hash, date, subject, files: [] };
} else if (current) {
// Name-status lines: "M\tfile.md", "A\tfile.md", "D\tfile.md", "R100\told.md\tnew.md"
const tabIdx = trimmed.indexOf("\t");
if (tabIdx !== -1) {
const statusCode = trimmed.slice(0, tabIdx);
const rest = trimmed.slice(tabIdx + 1);
// For renames, take the new path (after second tab)
const secondTab = rest.indexOf("\t");
const filePath = secondTab >= 0 ? rest.slice(secondTab + 1) : rest;
if (filePath.endsWith(".md")) {
let changeType: "added" | "modified" | "deleted" = "modified";
if (statusCode === "A") changeType = "added";
else if (statusCode === "D") changeType = "deleted";
current.files.push({ path: filePath, changeType });
}
}
}
}
if (current) commits.push(current);
return commits;
}
function extractTimelineExcerpt(compiledTruth: string, maxChars = 200): string {
// Pull the first few timeline-style bullet lines from compiled_truth
const lines = compiledTruth.split("\n");
const excerptLines: string[] = [];
for (const line of lines) {
if (/^- \*\*\d{4}-\d{2}-\d{2}/.test(line)) {
excerptLines.push(line);
if (excerptLines.length >= 3) break;
}
}
const excerpt = excerptLines.join("\n");
return excerpt.length > maxChars ? excerpt.slice(0, maxChars) + "…" : excerpt;
}
export function executeTimeline(
params: TimelineParams,
store: GBrainStore,
brainPath: string
): TimelineResult {
const startMs = Date.now();
const { since, until, entity, scope } = params;
const sinceDate = parseRelativeDate(since);
const commits = parseGitLog(brainPath, sinceDate, until);
// Flatten commits into per-file entries
const fileEntries: Array<{
date: string;
path: string;
changeType: "added" | "modified" | "deleted";
commitMessage: string;
}> = [];
for (const commit of commits) {
for (const file of commit.files) {
fileEntries.push({
date: commit.date,
path: file.path,
changeType: file.changeType,
commitMessage: commit.subject,
});
}
}
// Resolve entity filter to a set of relevant paths
let entityPaths: Set<string> | null = null;
if (entity) {
const matches = store.searchByName(entity);
if (matches.length > 0) {
const matched = matches[0];
if (matched) {
entityPaths = new Set<string>([matched.page.path]);
// Also include pages this entity mentions
const edges = store.getEdgesFrom(matched.page.id);
for (const e of edges) entityPaths.add(e.path);
}
}
}
const entries: TimelineEntry[] = [];
for (const entry of fileEntries) {
// Scope filter: directory prefix
if (scope) {
const normalizedPath = entry.path.replace(/\\/g, "/");
if (!normalizedPath.startsWith(scope + "/")) continue;
}
// Entity filter
if (entityPaths !== null && !entityPaths.has(entry.path)) continue;
// Look up the page in the index for title and type
const page = store.getPageByPath(entry.path);
let timelineExcerpt = "";
if (page && entry.changeType !== "deleted") {
timelineExcerpt = extractTimelineExcerpt(page.compiled_truth);
}
entries.push({
date: entry.date,
path: entry.path,
title: page?.title ?? entry.path,
type: page?.type ?? "unknown",
changeType: entry.changeType,
commitMessage: entry.commitMessage,
timelineExcerpt,
});
}
return { entries, queryTimeMs: Date.now() - startMs };
}
+34
View File
@@ -0,0 +1,34 @@
export interface GBrainConfig {
brainPath: string;
indexPath: string;
embeddingModel: string;
indexTimeline: boolean;
watchInterval: number;
chunkMaxTokens: number;
directories: string[];
excludeDirectories: string[];
}
export const DEFAULT_CONFIG: GBrainConfig = {
brainPath: "/data/brain",
indexPath: "/data/db/gbrain.db",
embeddingModel: "auto",
indexTimeline: false,
watchInterval: 30,
chunkMaxTokens: 1000,
directories: [],
excludeDirectories: [".raw", ".git", "node_modules"],
};
export function resolveConfig(raw: Record<string, unknown>): GBrainConfig {
return {
brainPath: (raw["brainPath"] as string) ?? DEFAULT_CONFIG.brainPath,
indexPath: (raw["indexPath"] as string) ?? DEFAULT_CONFIG.indexPath,
embeddingModel: (raw["embeddingModel"] as string) ?? DEFAULT_CONFIG.embeddingModel,
indexTimeline: (raw["indexTimeline"] as boolean) ?? DEFAULT_CONFIG.indexTimeline,
watchInterval: (raw["watchInterval"] as number) ?? DEFAULT_CONFIG.watchInterval,
chunkMaxTokens: (raw["chunkMaxTokens"] as number) ?? DEFAULT_CONFIG.chunkMaxTokens,
directories: (raw["directories"] as string[]) ?? DEFAULT_CONFIG.directories,
excludeDirectories: (raw["excludeDirectories"] as string[]) ?? DEFAULT_CONFIG.excludeDirectories,
};
}
+51
View File
@@ -0,0 +1,51 @@
declare module "openclaw/plugin-sdk/plugin-entry" {
export interface ToolContent {
type: "text";
text: string;
}
export interface ToolResult {
content: ToolContent[];
}
export interface PluginTool {
name: string;
description: string;
parameters: import("@sinclair/typebox").TObject;
execute: (id: string, params: Record<string, unknown>) => Promise<ToolResult>;
}
export interface ServiceContext {
config: Record<string, unknown>;
}
export interface PluginService {
id: string;
start: (ctx: ServiceContext) => Promise<void>;
stop: () => Promise<void>;
}
export interface CliCommand {
description: (desc: string) => CliCommand;
option: (flags: string, desc?: string, defaultValue?: string) => CliCommand;
action: (fn: (...args: unknown[]) => Promise<void>) => CliCommand;
command: (name: string) => CliCommand;
}
export interface CliProgram {
command: (name: string) => CliCommand;
}
export interface CliContext {
program: CliProgram;
}
export interface PluginApi {
registerTool: (tool: PluginTool) => void;
registerService: (service: PluginService) => void;
registerCli: (setup: (ctx: CliContext) => Promise<void>) => void;
config: Record<string, unknown>;
}
export function definePluginEntry(setup: (api: PluginApi) => void): void;
}
+20
View File
@@ -0,0 +1,20 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"outDir": "dist",
"rootDir": "src",
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"allowImportingTsExtensions": false,
"verbatimModuleSyntax": false
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist", "test"]
}