Compare commits

...
Author SHA1 Message Date
Wintermute acdee0905a feat: code indexing + multi-repo support
Tree-sitter-based code chunker for TS/JS/Python/Ruby/Go.
Splits code at semantic boundaries (functions, classes, types, exports).
Each chunk includes structured header for embedding context.

Multi-repo config: `gbrain repos add/list/remove`, `gbrain sync --all`.
Strategy-aware sync: markdown (default), code, or auto.
New PageType 'code' for code file pages.

Backward compatible: no config changes = existing behavior preserved.
All 37 sync tests pass, typecheck clean.
2026-04-22 15:34:15 +00:00
11 changed files with 891 additions and 16 deletions
+6
View File
@@ -14,6 +14,8 @@
"openai": "^4.0.0",
"pgvector": "^0.2.0",
"postgres": "^3.4.0",
"tree-sitter-wasms": "0.1.13",
"web-tree-sitter": "0.22.6",
},
"devDependencies": {
"@types/bun": "latest",
@@ -453,6 +455,8 @@
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
"tree-sitter-wasms": ["tree-sitter-wasms@0.1.13", "", { "dependencies": { "tree-sitter-wasms": "^0.1.11" } }, "sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ=="],
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
@@ -467,6 +471,8 @@
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
"web-tree-sitter": ["web-tree-sitter@0.22.6", "", {}, "sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q=="],
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
+3 -1
View File
@@ -44,7 +44,9 @@
"marked": "^18.0.0",
"openai": "^4.0.0",
"pgvector": "^0.2.0",
"postgres": "^3.4.0"
"postgres": "^3.4.0",
"tree-sitter-wasms": "0.1.13",
"web-tree-sitter": "0.22.6"
},
"devDependencies": {
"@types/bun": "latest",
+12 -1
View File
@@ -19,7 +19,7 @@ for (const op of operations) {
}
// CLI-only commands that bypass the operation layer
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'dream', 'check-resolvable']);
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'dream', 'check-resolvable', 'repos']);
async function main() {
// Parse global flags (--quiet / --progress-json / --progress-interval)
@@ -315,6 +315,11 @@ async function handleCliOnly(command: string, args: string[]) {
await runCheckResolvable(args);
return;
}
if (command === 'repos') {
const { handleRepos } = await import('./commands/repos.ts');
await handleRepos(args);
return;
}
if (command === 'report') {
const { runReport } = await import('./commands/report.ts');
await runReport(args);
@@ -586,6 +591,12 @@ TOOLS
check-resolvable [--json] [--fix] Validate skill tree (reachability/MECE/DRY)
report --type <name> --content ... Save timestamped report to brain/reports/
MULTI-REPO
repos list Show configured repos
repos add <path> [--name N] Add a repo [--strategy markdown|code|auto]
repos remove <name> Remove a repo
sync --all Sync all configured repos
JOBS (Minions)
jobs submit <name> [--params JSON] Submit background job [--follow] [--dry-run]
jobs list [--status S] [--limit N] List jobs
+121
View File
@@ -0,0 +1,121 @@
/**
* CLI: gbrain repos list|add|remove
* Multi-repo management for code + knowledge indexing.
*/
import { resolve } from 'path';
import { existsSync } from 'fs';
import {
loadRepoConfigs,
addRepoConfig,
removeRepoConfig,
normalizeRepoName,
type RepoStrategy,
} from '../core/multi-repo.ts';
export async function handleRepos(args: string[]): Promise<void> {
const sub = args[0];
if (!sub || sub === 'list') {
return reposList();
}
if (sub === 'add') {
return reposAdd(args.slice(1));
}
if (sub === 'remove' || sub === 'rm') {
return reposRemove(args.slice(1));
}
console.error(`Unknown repos subcommand: ${sub}`);
console.error('Usage: gbrain repos [list|add|remove]');
process.exit(1);
}
function reposList(): void {
const repos = loadRepoConfigs();
if (repos.length === 0) {
console.log('No repos configured. Use `gbrain repos add <path>` to add one.');
return;
}
console.log(`${repos.length} repo(s) configured:\n`);
for (const repo of repos) {
const enabled = repo.syncEnabled !== false ? '✓' : '✗';
const includes = repo.include?.length ? ` include=[${repo.include.join(',')}]` : '';
const excludes = repo.exclude?.length ? ` exclude=[${repo.exclude.join(',')}]` : '';
console.log(` ${enabled} ${repo.name} (${repo.strategy}) → ${repo.path}${includes}${excludes}`);
}
}
function reposAdd(args: string[]): void {
if (args.length === 0) {
console.error('Usage: gbrain repos add <path> [--name <name>] [--strategy markdown|code|auto]');
process.exit(1);
}
const repoPath = resolve(args[0]);
if (!existsSync(repoPath)) {
console.error(`Path does not exist: ${repoPath}`);
process.exit(1);
}
let name = normalizeRepoName(repoPath);
let strategy: RepoStrategy = 'auto';
const include: string[] = [];
const exclude: string[] = [];
for (let i = 1; i < args.length; i++) {
if (args[i] === '--name' && args[i + 1]) {
name = args[++i];
} else if (args[i] === '--strategy' && args[i + 1]) {
const s = args[++i];
if (s === 'markdown' || s === 'code' || s === 'auto') {
strategy = s;
} else {
console.error(`Invalid strategy: ${s}. Must be markdown, code, or auto.`);
process.exit(1);
}
} else if (args[i] === '--include' && args[i + 1]) {
include.push(args[++i]);
} else if (args[i] === '--exclude' && args[i + 1]) {
exclude.push(args[++i]);
}
}
try {
const repos = addRepoConfig({
path: repoPath,
name,
strategy,
include: include.length > 0 ? include : undefined,
exclude: exclude.length > 0 ? exclude : undefined,
syncEnabled: true,
});
console.log(`Added repo "${name}" (${strategy}) → ${repoPath}`);
console.log(`${repos.length} repo(s) total. Run \`gbrain sync --all\` to index.`);
} catch (e: unknown) {
console.error(e instanceof Error ? e.message : String(e));
process.exit(1);
}
}
function reposRemove(args: string[]): void {
if (args.length === 0) {
console.error('Usage: gbrain repos remove <name>');
process.exit(1);
}
const name = args[0];
try {
const repos = removeRepoConfig(name);
console.log(`Removed repo "${name}". ${repos.length} repo(s) remaining.`);
} catch (e: unknown) {
console.error(e instanceof Error ? e.message : String(e));
process.exit(1);
}
}
+41 -7
View File
@@ -41,6 +41,8 @@ export interface SyncOpts {
skipFailed?: boolean;
/** Bug 9 — re-attempt unacknowledged failures explicitly (CLI --retry-failed). */
retryFailed?: boolean;
/** Multi-repo: sync strategy override (markdown, code, auto). */
strategy?: 'markdown' | 'code' | 'auto';
}
function git(repoPath: string, ...args: string[]): string {
@@ -127,16 +129,17 @@ export async function performSync(engine: BrainEngine, opts: SyncOpts): Promise<
const diffOutput = git(repoPath, 'diff', '--name-status', '-M', `${lastCommit}..${headCommit}`);
const manifest = buildSyncManifest(diffOutput);
// Filter to syncable files
// Filter to syncable files (strategy-aware)
const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined;
const filtered: SyncManifest = {
added: manifest.added.filter(p => isSyncable(p)),
modified: manifest.modified.filter(p => isSyncable(p)),
deleted: manifest.deleted.filter(p => isSyncable(p)),
renamed: manifest.renamed.filter(r => isSyncable(r.to)),
added: manifest.added.filter(p => isSyncable(p, syncOpts)),
modified: manifest.modified.filter(p => isSyncable(p, syncOpts)),
deleted: manifest.deleted.filter(p => isSyncable(p, syncOpts)),
renamed: manifest.renamed.filter(r => isSyncable(r.to, syncOpts)),
};
// Delete pages that became un-syncable (modified but filtered out)
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p));
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p, syncOpts));
for (const path of unsyncableModified) {
const slug = pathToSlug(path);
try {
@@ -481,8 +484,39 @@ export async function runSync(engine: BrainEngine, args: string[]) {
const noEmbed = args.includes('--no-embed');
const skipFailed = args.includes('--skip-failed');
const retryFailed = args.includes('--retry-failed');
const syncAll = args.includes('--all');
const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined;
const opts: SyncOpts = { repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed };
// Multi-repo: --all syncs all configured repos
if (syncAll) {
const { loadRepoConfigs } = await import('../core/multi-repo.ts');
const repos = loadRepoConfigs();
if (repos.length === 0) {
console.log('No repos configured. Use `gbrain repos add <path>` first.');
return;
}
for (const repo of repos) {
if (repo.syncEnabled === false) {
console.log(`Skipping disabled repo: ${repo.name}`);
continue;
}
console.log(`\n--- Syncing repo: ${repo.name} (${repo.strategy}) ---`);
const repoOpts: SyncOpts = {
repoPath: repo.path,
dryRun, full, noPull, noEmbed, skipFailed, retryFailed,
strategy: repo.strategy,
};
try {
const result = await performSync(engine, repoOpts);
printSyncResult(result);
} catch (e: unknown) {
console.error(`Error syncing ${repo.name}: ${e instanceof Error ? e.message : String(e)}`);
}
}
return;
}
const opts: SyncOpts = { repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed, strategy: strategyArg };
// Bug 9 — --retry-failed: before running normal sync, clear acknowledgment
// flags so the sync picks them up as fresh work. The actual re-attempt
+408
View File
@@ -0,0 +1,408 @@
/**
* Code Chunker — Tree-Sitter-Based Semantic Code Splitting
*
* Uses web-tree-sitter (WASM) to parse code files into AST, then extracts
* semantic units (functions, classes, types, exports) as chunks.
*
* Each chunk includes a structured header with language, file path, line range,
* and symbol name — so embeddings capture both context and code content.
*
* Supports: TypeScript, TSX, JavaScript, Python, Ruby, Go.
* Falls back to recursive text chunker for unsupported languages.
*/
import { chunkText as recursiveChunk } from './recursive.ts';
// Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
let Parser: typeof import('web-tree-sitter') | null = null;
async function getParser(): Promise<typeof import('web-tree-sitter')> {
if (!Parser) {
Parser = (await import('web-tree-sitter')).default || await import('web-tree-sitter');
}
return Parser;
}
export type SupportedCodeLanguage = 'typescript' | 'tsx' | 'javascript' | 'python' | 'ruby' | 'go';
export interface CodeChunkMetadata {
symbolName: string | null;
symbolType: string;
filePath: string;
language: SupportedCodeLanguage;
startLine: number;
endLine: number;
}
export interface CodeChunk {
text: string;
index: number;
metadata: CodeChunkMetadata;
}
export interface CodeChunkOptions {
chunkSizeTokens?: number;
largeChunkThresholdTokens?: number;
fallbackChunkSizeWords?: number;
fallbackOverlapWords?: number;
}
const GRAMMAR_FILES: Record<SupportedCodeLanguage, string> = {
typescript: 'tree-sitter-typescript.wasm',
tsx: 'tree-sitter-tsx.wasm',
javascript: 'tree-sitter-javascript.wasm',
python: 'tree-sitter-python.wasm',
ruby: 'tree-sitter-ruby.wasm',
go: 'tree-sitter-go.wasm',
};
const TOP_LEVEL_TYPES: Record<SupportedCodeLanguage, Set<string>> = {
typescript: new Set([
'function_declaration',
'class_declaration',
'abstract_class_declaration',
'interface_declaration',
'type_alias_declaration',
'enum_declaration',
'lexical_declaration',
'variable_declaration',
'export_statement',
]),
tsx: new Set([
'function_declaration',
'class_declaration',
'interface_declaration',
'type_alias_declaration',
'enum_declaration',
'lexical_declaration',
'variable_declaration',
'export_statement',
]),
javascript: new Set([
'function_declaration',
'class_declaration',
'lexical_declaration',
'variable_declaration',
'export_statement',
]),
python: new Set([
'function_definition',
'class_definition',
'import_statement',
'import_from_statement',
'assignment',
]),
ruby: new Set([
'class',
'module',
'method',
'singleton_method',
'assignment',
]),
go: new Set([
'function_declaration',
'method_declaration',
'type_declaration',
'const_declaration',
'var_declaration',
'import_declaration',
]),
};
const BODY_NODE_TYPES = new Set([
'statement_block',
'block',
'class_body',
'module_body',
'body_statement',
'body',
]);
let initDone = false;
let initPromise: Promise<void> | null = null;
const languageCache = new Map<SupportedCodeLanguage, any>();
// ---------- Public API ----------
export function detectCodeLanguage(filePath: string): SupportedCodeLanguage | null {
const lower = filePath.toLowerCase();
if (lower.endsWith('.tsx')) return 'tsx';
if (lower.endsWith('.ts')) return 'typescript';
if (lower.endsWith('.js') || lower.endsWith('.jsx') || lower.endsWith('.mjs') || lower.endsWith('.cjs')) return 'javascript';
if (lower.endsWith('.py')) return 'python';
if (lower.endsWith('.rb')) return 'ruby';
if (lower.endsWith('.go')) return 'go';
return null;
}
export async function chunkCodeText(
source: string,
filePath: string,
opts: CodeChunkOptions = {},
): Promise<CodeChunk[]> {
const language = detectCodeLanguage(filePath);
if (!language) {
return fallbackChunks(source, filePath, 'javascript', opts);
}
if (!source.trim()) return [];
const largeThreshold = opts.largeChunkThresholdTokens ?? 1000;
const chunkTarget = opts.chunkSizeTokens ?? 300;
try {
await ensureInit();
const P = await getParser();
const parser = new (P as any)();
const grammar = await loadLanguage(language);
parser.setLanguage(grammar);
const tree = parser.parse(source);
if (!tree) {
parser.delete();
return fallbackChunks(source, filePath, language, opts);
}
const root = tree.rootNode;
const topLevelTypes = TOP_LEVEL_TYPES[language];
const semanticNodes = root.namedChildren.filter((n: any) => topLevelTypes.has(n.type));
if (semanticNodes.length === 0) {
tree.delete();
parser.delete();
return fallbackChunks(source, filePath, language, opts);
}
const chunks: CodeChunk[] = [];
for (const node of semanticNodes) {
const symbolName = extractSymbolName(node);
const symbolType = normalizeSymbolType(node.type);
const nodeText = source.slice(node.startIndex, node.endIndex).trim();
if (!nodeText) continue;
if (estimateTokens(nodeText) <= largeThreshold) {
chunks.push(buildChunk({
body: nodeText, filePath, language, symbolName, symbolType,
startLine: node.startPosition.row + 1,
endLine: node.endPosition.row + 1,
index: chunks.length,
}));
continue;
}
// Split very large nodes at nested block boundaries
const subRanges = splitLargeNode(node, source, chunkTarget);
if (subRanges.length === 0) {
chunks.push(buildChunk({
body: nodeText, filePath, language, symbolName, symbolType,
startLine: node.startPosition.row + 1,
endLine: node.endPosition.row + 1,
index: chunks.length,
}));
continue;
}
for (const range of subRanges) {
const body = source.slice(range.startIndex, range.endIndex).trim();
if (!body) continue;
chunks.push(buildChunk({
body, filePath, language, symbolName, symbolType,
startLine: range.startLine, endLine: range.endLine,
index: chunks.length,
}));
}
}
tree.delete();
parser.delete();
return chunks.length > 0 ? chunks : fallbackChunks(source, filePath, language, opts);
} catch {
return fallbackChunks(source, filePath, language, opts);
}
}
// ---------- Internals ----------
function fallbackChunks(
source: string,
filePath: string,
language: SupportedCodeLanguage,
opts: CodeChunkOptions,
): CodeChunk[] {
const size = opts.fallbackChunkSizeWords ?? 300;
const overlap = opts.fallbackOverlapWords ?? 50;
return recursiveChunk(source, { chunkSize: size, chunkOverlap: overlap }).map((chunk, index) =>
buildChunk({
body: chunk.text, filePath, language,
symbolName: null, symbolType: 'module',
startLine: 1, endLine: countLines(chunk.text),
index,
}),
);
}
function buildChunk(input: {
body: string;
filePath: string;
language: SupportedCodeLanguage;
symbolName: string | null;
symbolType: string;
startLine: number;
endLine: number;
index: number;
}): CodeChunk {
const symbol = input.symbolName ? `${input.symbolType} ${input.symbolName}` : input.symbolType;
const header = `[${displayLang(input.language)}] ${input.filePath}:${input.startLine}-${input.endLine} ${symbol}`;
return {
index: input.index,
text: `${header}\n\n${input.body}`,
metadata: {
symbolName: input.symbolName,
symbolType: input.symbolType,
filePath: input.filePath,
language: input.language,
startLine: input.startLine,
endLine: input.endLine,
},
};
}
interface SplitRange {
startIndex: number;
endIndex: number;
startLine: number;
endLine: number;
}
function splitLargeNode(node: any, source: string, chunkTarget: number): SplitRange[] {
const body =
node.childForFieldName('body') ||
node.namedChildren.find((c: any) => BODY_NODE_TYPES.has(c.type)) ||
null;
if (!body || body.namedChildren.length < 2) return [];
const children = body.namedChildren.filter((c: any) => !c.isExtra);
if (children.length < 2) return [];
const ranges: SplitRange[] = [];
let curStart = children[0].startIndex;
let curStartLine = children[0].startPosition.row + 1;
let curEnd = children[0].endIndex;
let curEndLine = children[0].endPosition.row + 1;
let curTokens = estimateTokens(source.slice(curStart, curEnd));
for (let i = 1; i < children.length; i++) {
const child = children[i];
const childTokens = estimateTokens(source.slice(child.startIndex, child.endIndex));
if (curTokens + childTokens > Math.ceil(chunkTarget * 1.5)) {
ranges.push({ startIndex: curStart, endIndex: curEnd, startLine: curStartLine, endLine: curEndLine });
curStart = child.startIndex;
curStartLine = child.startPosition.row + 1;
curEnd = child.endIndex;
curEndLine = child.endPosition.row + 1;
curTokens = childTokens;
} else {
curEnd = child.endIndex;
curEndLine = child.endPosition.row + 1;
curTokens += childTokens;
}
}
ranges.push({ startIndex: curStart, endIndex: curEnd, startLine: curStartLine, endLine: curEndLine });
return ranges;
}
function extractSymbolName(node: any): string | null {
const directName = node.childForFieldName('name');
if (directName?.text?.trim()) return sanitize(directName.text);
const declaration = node.childForFieldName('declaration');
if (declaration) {
const nested = extractSymbolName(declaration);
if (nested) return nested;
}
for (const child of node.namedChildren) {
if (child.type.endsWith('identifier') || child.type === 'constant') {
const v = sanitize(child.text);
if (v) return v;
}
}
return null;
}
function normalizeSymbolType(type: string): string {
if (type.includes('function') || type === 'method' || type === 'singleton_method') return 'function';
if (type.includes('class')) return 'class';
if (type.includes('interface')) return 'interface';
if (type.includes('type_alias')) return 'type';
if (type.includes('enum')) return 'enum';
if (type.includes('module')) return 'module';
if (type.includes('import')) return 'import';
return type.replace(/_/g, ' ');
}
function sanitize(name: string): string {
return name.replace(/[\n\r\t]+/g, ' ').replace(/\s+/g, ' ').trim();
}
function estimateTokens(text: string): number {
return Math.max(1, Math.ceil(text.length / 4));
}
function displayLang(lang: SupportedCodeLanguage): string {
const map: Record<SupportedCodeLanguage, string> = {
typescript: 'TypeScript', tsx: 'TSX', javascript: 'JavaScript',
python: 'Python', ruby: 'Ruby', go: 'Go',
};
return map[lang];
}
function countLines(text: string): number {
return text ? text.split('\n').length : 0;
}
// ---------- Tree-sitter init ----------
async function ensureInit(): Promise<void> {
if (initDone) return;
if (!initPromise) {
initPromise = (async () => {
const P = await getParser();
// v0.22.x: init takes locateFile for the WASM module
const wasmPath = new URL('../../../node_modules/web-tree-sitter/tree-sitter.wasm', import.meta.url);
let resolved: string;
try {
const { fileURLToPath } = await import('url');
resolved = fileURLToPath(wasmPath);
} catch {
resolved = wasmPath.pathname;
}
await (P as any).init({ locateFile: () => resolved });
initDone = true;
})();
}
await initPromise;
}
async function loadLanguage(language: SupportedCodeLanguage): Promise<any> {
if (languageCache.has(language)) return languageCache.get(language);
const P = await getParser();
const grammarUrl = new URL(
`../../../node_modules/tree-sitter-wasms/out/${GRAMMAR_FILES[language]}`,
import.meta.url,
);
let resolved: string;
try {
const { fileURLToPath } = await import('url');
resolved = fileURLToPath(grammarUrl);
} catch {
resolved = grammarUrl.pathname;
}
const lang = await (P as any).Language.load(resolved);
languageCache.set(language, lang);
return lang;
}
+8
View File
@@ -27,6 +27,14 @@ export interface GBrainConfig {
engine: 'postgres' | 'pglite';
database_url?: string;
database_path?: string;
repos?: Array<{
path: string;
name: string;
strategy: 'markdown' | 'code' | 'auto';
include?: string[];
exclude?: string[];
syncEnabled?: boolean;
}>;
openai_api_key?: string;
anthropic_api_key?: string;
/**
+86 -1
View File
@@ -1,10 +1,12 @@
import { readFileSync, statSync, lstatSync } from 'fs';
import { basename } from 'path';
import { createHash } from 'crypto';
import type { BrainEngine } from './engine.ts';
import { parseMarkdown } from './markdown.ts';
import { chunkText } from './chunkers/recursive.ts';
import { chunkCodeText, detectCodeLanguage } from './chunkers/code.ts';
import { embedBatch } from './embedding.ts';
import { slugifyPath } from './sync.ts';
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
import type { ChunkInput, PageType } from './types.ts';
/**
@@ -184,6 +186,12 @@ export async function importFromFile(
}
const content = readFileSync(filePath, 'utf-8');
// Route code files through the code import path
if (isCodeFilePath(relativePath)) {
return importCodeFile(engine, relativePath, content, opts);
}
const parsed = parseMarkdown(content, relativePath);
// Enforce path-authoritative slug. parseMarkdown prefers frontmatter.slug over
@@ -206,6 +214,83 @@ export async function importFromFile(
return importFromContent(engine, expectedSlug, content, opts);
}
/**
* Import a code file. Bypasses markdown parsing entirely.
* Uses tree-sitter code chunker for semantic splitting.
* Page type is 'code', slug includes file extension.
*/
export async function importCodeFile(
engine: BrainEngine,
relativePath: string,
content: string,
opts: { noEmbed?: boolean } = {},
): Promise<ImportResult> {
const slug = slugifyCodePath(relativePath);
const lang = detectCodeLanguage(relativePath) || 'unknown';
const title = `${relativePath} (${lang})`;
const byteLength = Buffer.byteLength(content, 'utf-8');
if (byteLength > MAX_FILE_SIZE) {
return { slug, status: 'skipped', chunks: 0, error: `Code file too large (${byteLength} bytes)` };
}
// Hash for idempotency
const hash = createHash('sha256')
.update(JSON.stringify({ title, type: 'code', content, lang }))
.digest('hex');
const existing = await engine.getPage(slug);
if (existing?.content_hash === hash) {
return { slug, status: 'skipped', chunks: 0 };
}
// Chunk via tree-sitter code chunker
const codeChunks = await chunkCodeText(content, relativePath);
const chunks: ChunkInput[] = codeChunks.map((c, i) => ({
chunk_index: i,
chunk_text: c.text,
chunk_source: 'compiled_truth' as const,
}));
// Embed
if (!opts.noEmbed && chunks.length > 0) {
try {
const embeddings = await embedBatch(chunks.map(c => c.chunk_text));
for (let i = 0; i < chunks.length; i++) {
chunks[i].embedding = embeddings[i];
chunks[i].token_count = Math.ceil(chunks[i].chunk_text.length / 4);
}
} catch (e: unknown) {
console.warn(`[gbrain] embedding failed for code file ${slug}: ${e instanceof Error ? e.message : String(e)}`);
}
}
// Store
await engine.transaction(async (tx) => {
if (existing) await tx.createVersion(slug);
await tx.putPage(slug, {
type: 'code' as PageType,
title,
compiled_truth: content,
timeline: '',
frontmatter: { language: lang, file: relativePath },
content_hash: hash,
});
await tx.addTag(slug, 'code');
await tx.addTag(slug, lang);
if (chunks.length > 0) {
await tx.upsertChunks(slug, chunks);
} else {
await tx.deleteChunks(slug);
}
});
return { slug, status: 'imported', chunks: chunks.length };
}
// Backward compat
export const importFile = importFromFile;
export type ImportFileResult = ImportResult;
+120
View File
@@ -0,0 +1,120 @@
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
import { resolve } from 'path';
import { configDir, configPath } from './config.ts';
export type RepoStrategy = 'markdown' | 'code' | 'auto';
export interface RepoConfig {
path: string;
name: string;
strategy: RepoStrategy;
include?: string[];
exclude?: string[];
syncEnabled?: boolean;
}
interface RawConfig {
[key: string]: unknown;
repos?: unknown;
}
function readRawConfig(): RawConfig {
try {
const raw = readFileSync(configPath(), 'utf-8');
return JSON.parse(raw) as RawConfig;
} catch {
return {};
}
}
function writeRawConfig(config: RawConfig): void {
mkdirSync(configDir(), { recursive: true });
writeFileSync(configPath(), JSON.stringify(config, null, 2) + '\n', { mode: 0o600 });
try {
chmodSync(configPath(), 0o600);
} catch {
// chmod may fail on some platforms
}
}
export function normalizeRepoName(repoPath: string): string {
const normalized = repoPath.replace(/\\/g, '/').replace(/\/+$/g, '');
const parts = normalized.split('/').filter(Boolean);
return parts[parts.length - 1] || 'repo';
}
export function loadRepoConfigs(): RepoConfig[] {
if (!existsSync(configPath())) return [];
const parsed = readRawConfig();
if (!Array.isArray(parsed.repos)) return [];
const repos: RepoConfig[] = [];
for (const candidate of parsed.repos) {
if (!candidate || typeof candidate !== 'object') continue;
const row = candidate as Record<string, unknown>;
if (typeof row.path !== 'string' || typeof row.name !== 'string') continue;
const strategy = row.strategy;
const normalizedStrategy: RepoStrategy = strategy === 'code' || strategy === 'auto' ? strategy : 'markdown';
repos.push({
path: resolve(row.path),
name: row.name,
strategy: normalizedStrategy,
include: Array.isArray(row.include) ? row.include.map(String) : undefined,
exclude: Array.isArray(row.exclude) ? row.exclude.map(String) : undefined,
syncEnabled: typeof row.syncEnabled === 'boolean' ? row.syncEnabled : true,
});
}
return repos;
}
export function saveRepoConfigs(repos: RepoConfig[]): void {
const parsed = readRawConfig();
parsed.repos = repos.map((repo) => ({
path: resolve(repo.path),
name: repo.name,
strategy: repo.strategy,
...(repo.include && repo.include.length > 0 ? { include: repo.include } : {}),
...(repo.exclude && repo.exclude.length > 0 ? { exclude: repo.exclude } : {}),
...(repo.syncEnabled === false ? { syncEnabled: false } : {}),
}));
writeRawConfig(parsed);
}
export function addRepoConfig(repo: RepoConfig): RepoConfig[] {
const repos = loadRepoConfigs();
const normalized: RepoConfig = {
...repo,
path: resolve(repo.path),
name: repo.name || normalizeRepoName(repo.path),
strategy: repo.strategy || 'auto',
syncEnabled: repo.syncEnabled ?? true,
};
const nameTaken = repos.find((r) => r.name === normalized.name);
if (nameTaken) {
throw new Error(`Repo name already exists: ${normalized.name}`);
}
const pathTaken = repos.find((r) => resolve(r.path) === normalized.path);
if (pathTaken) {
throw new Error(`Repo path already configured: ${normalized.path}`);
}
const updated = [...repos, normalized];
saveRepoConfigs(updated);
return updated;
}
export function removeRepoConfig(name: string): RepoConfig[] {
const repos = loadRepoConfigs();
const next = repos.filter((r) => r.name !== name);
if (next.length === repos.length) {
throw new Error(`Repo not found: ${name}`);
}
saveRepoConfigs(next);
return next;
}
+85 -5
View File
@@ -24,6 +24,16 @@ export interface RawManifestEntry {
oldPath?: string;
}
export type SyncStrategy = 'markdown' | 'code' | 'auto';
interface SyncableOptions {
strategy?: SyncStrategy;
include?: string[];
exclude?: string[];
}
const CODE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py', '.rb', '.go']);
/**
* Parse the output of `git diff --name-status -M LAST..HEAD` into structured entries.
*
@@ -72,12 +82,60 @@ export function buildSyncManifest(gitDiffOutput: string): SyncManifest {
return manifest;
}
export function isCodeFilePath(path: string): boolean {
const lower = path.toLowerCase();
for (const ext of CODE_EXTENSIONS) {
if (lower.endsWith(ext)) return true;
}
return false;
}
function isMarkdownFilePath(path: string): boolean {
return path.endsWith('.md') || path.endsWith('.mdx');
}
function isAllowedByStrategy(path: string, strategy: SyncStrategy): boolean {
if (strategy === 'markdown') return isMarkdownFilePath(path);
if (strategy === 'code') return isCodeFilePath(path);
return isMarkdownFilePath(path) || isCodeFilePath(path);
}
function globToRegex(pattern: string): RegExp {
let regex = '^';
for (let i = 0; i < pattern.length; i++) {
const ch = pattern[i];
if (ch === '*') {
const next = pattern[i + 1];
if (next === '*') {
regex += '.*';
i++;
} else {
regex += '[^/]*';
}
continue;
}
if (ch === '?') { regex += '.'; continue; }
if ('\\.[]{}()+-^$|'.includes(ch)) { regex += `\\${ch}`; continue; }
regex += ch;
}
regex += '$';
return new RegExp(regex);
}
function matchesAnyGlob(path: string, patterns?: string[]): boolean {
if (!patterns || patterns.length === 0) return false;
const normalized = path.replace(/\\/g, '/');
return patterns.some((pattern) => globToRegex(pattern).test(normalized));
}
/**
* Filter a file path to determine if it should be synced to GBrain.
* Strategy-aware: 'markdown' (default) = .md/.mdx only, 'code' = code files only, 'auto' = both.
*/
export function isSyncable(path: string): boolean {
// Must be .md or .mdx
if (!path.endsWith('.md') && !path.endsWith('.mdx')) return false;
export function isSyncable(path: string, opts: SyncableOptions = {}): boolean {
const strategy = opts.strategy || 'markdown';
if (!isAllowedByStrategy(path, strategy)) return false;
// Skip hidden directories
if (path.split('/').some(p => p.startsWith('.'))) return false;
@@ -93,6 +151,9 @@ export function isSyncable(path: string): boolean {
// Skip ops/ directory
if (path.startsWith('ops/')) return false;
if (opts.include && opts.include.length > 0 && !matchesAnyGlob(path, opts.include)) return false;
if (opts.exclude && opts.exclude.length > 0 && matchesAnyGlob(path, opts.exclude)) return false;
return true;
}
@@ -125,11 +186,30 @@ export function slugifyPath(filePath: string): string {
return path.split('/').map(slugifySegment).filter(Boolean).join('/');
}
/**
* Slugify a code file path: flatten into a single slug segment with dots → hyphens.
* e.g. 'src/core/chunkers/code.ts' → 'src-core-chunkers-code-ts'
*/
export function slugifyCodePath(filePath: string): string {
let path = filePath.replace(/\\/g, '/');
path = path.replace(/^\.?\//, '');
return path
.split('/')
.map(segment => slugifySegment(segment.replace(/\./g, '-')))
.filter(Boolean)
.join('-');
}
/**
* Convert a repo-relative file path to a GBrain page slug.
*/
export function pathToSlug(filePath: string, repoPrefix?: string): string {
let slug = slugifyPath(filePath);
export function pathToSlug(
filePath: string,
repoPrefix?: string,
options: { pageKind?: 'markdown' | 'code' } = {},
): string {
const pageKind = options.pageKind || 'markdown';
let slug = pageKind === 'code' ? slugifyCodePath(filePath) : slugifyPath(filePath);
if (repoPrefix) slug = `${repoPrefix}/${slug}`;
return slug.toLowerCase();
}
+1 -1
View File
@@ -1,5 +1,5 @@
// Page types
export type PageType = 'person' | 'company' | 'deal' | 'yc' | 'civic' | 'project' | 'concept' | 'source' | 'media' | 'writing' | 'analysis' | 'guide' | 'hardware' | 'architecture' | 'meeting' | 'note';
export type PageType = 'person' | 'company' | 'deal' | 'yc' | 'civic' | 'project' | 'concept' | 'source' | 'media' | 'writing' | 'analysis' | 'guide' | 'hardware' | 'architecture' | 'meeting' | 'note' | 'code';
export interface Page {
id: number;