mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
acdee0905a |
@@ -14,6 +14,8 @@
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
@@ -453,6 +455,8 @@
|
||||
|
||||
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
|
||||
|
||||
"tree-sitter-wasms": ["tree-sitter-wasms@0.1.13", "", { "dependencies": { "tree-sitter-wasms": "^0.1.11" } }, "sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ=="],
|
||||
|
||||
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
|
||||
@@ -467,6 +471,8 @@
|
||||
|
||||
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
|
||||
|
||||
"web-tree-sitter": ["web-tree-sitter@0.22.6", "", {}, "sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0"
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
|
||||
+12
-1
@@ -19,7 +19,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'dream', 'check-resolvable']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'dream', 'check-resolvable', 'repos']);
|
||||
|
||||
async function main() {
|
||||
// Parse global flags (--quiet / --progress-json / --progress-interval)
|
||||
@@ -315,6 +315,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runCheckResolvable(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'repos') {
|
||||
const { handleRepos } = await import('./commands/repos.ts');
|
||||
await handleRepos(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'report') {
|
||||
const { runReport } = await import('./commands/report.ts');
|
||||
await runReport(args);
|
||||
@@ -586,6 +591,12 @@ TOOLS
|
||||
check-resolvable [--json] [--fix] Validate skill tree (reachability/MECE/DRY)
|
||||
report --type <name> --content ... Save timestamped report to brain/reports/
|
||||
|
||||
MULTI-REPO
|
||||
repos list Show configured repos
|
||||
repos add <path> [--name N] Add a repo [--strategy markdown|code|auto]
|
||||
repos remove <name> Remove a repo
|
||||
sync --all Sync all configured repos
|
||||
|
||||
JOBS (Minions)
|
||||
jobs submit <name> [--params JSON] Submit background job [--follow] [--dry-run]
|
||||
jobs list [--status S] [--limit N] List jobs
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* CLI: gbrain repos list|add|remove
|
||||
* Multi-repo management for code + knowledge indexing.
|
||||
*/
|
||||
|
||||
import { resolve } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import {
|
||||
loadRepoConfigs,
|
||||
addRepoConfig,
|
||||
removeRepoConfig,
|
||||
normalizeRepoName,
|
||||
type RepoStrategy,
|
||||
} from '../core/multi-repo.ts';
|
||||
|
||||
export async function handleRepos(args: string[]): Promise<void> {
|
||||
const sub = args[0];
|
||||
|
||||
if (!sub || sub === 'list') {
|
||||
return reposList();
|
||||
}
|
||||
|
||||
if (sub === 'add') {
|
||||
return reposAdd(args.slice(1));
|
||||
}
|
||||
|
||||
if (sub === 'remove' || sub === 'rm') {
|
||||
return reposRemove(args.slice(1));
|
||||
}
|
||||
|
||||
console.error(`Unknown repos subcommand: ${sub}`);
|
||||
console.error('Usage: gbrain repos [list|add|remove]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function reposList(): void {
|
||||
const repos = loadRepoConfigs();
|
||||
|
||||
if (repos.length === 0) {
|
||||
console.log('No repos configured. Use `gbrain repos add <path>` to add one.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`${repos.length} repo(s) configured:\n`);
|
||||
for (const repo of repos) {
|
||||
const enabled = repo.syncEnabled !== false ? '✓' : '✗';
|
||||
const includes = repo.include?.length ? ` include=[${repo.include.join(',')}]` : '';
|
||||
const excludes = repo.exclude?.length ? ` exclude=[${repo.exclude.join(',')}]` : '';
|
||||
console.log(` ${enabled} ${repo.name} (${repo.strategy}) → ${repo.path}${includes}${excludes}`);
|
||||
}
|
||||
}
|
||||
|
||||
function reposAdd(args: string[]): void {
|
||||
if (args.length === 0) {
|
||||
console.error('Usage: gbrain repos add <path> [--name <name>] [--strategy markdown|code|auto]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const repoPath = resolve(args[0]);
|
||||
|
||||
if (!existsSync(repoPath)) {
|
||||
console.error(`Path does not exist: ${repoPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let name = normalizeRepoName(repoPath);
|
||||
let strategy: RepoStrategy = 'auto';
|
||||
const include: string[] = [];
|
||||
const exclude: string[] = [];
|
||||
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
if (args[i] === '--name' && args[i + 1]) {
|
||||
name = args[++i];
|
||||
} else if (args[i] === '--strategy' && args[i + 1]) {
|
||||
const s = args[++i];
|
||||
if (s === 'markdown' || s === 'code' || s === 'auto') {
|
||||
strategy = s;
|
||||
} else {
|
||||
console.error(`Invalid strategy: ${s}. Must be markdown, code, or auto.`);
|
||||
process.exit(1);
|
||||
}
|
||||
} else if (args[i] === '--include' && args[i + 1]) {
|
||||
include.push(args[++i]);
|
||||
} else if (args[i] === '--exclude' && args[i + 1]) {
|
||||
exclude.push(args[++i]);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const repos = addRepoConfig({
|
||||
path: repoPath,
|
||||
name,
|
||||
strategy,
|
||||
include: include.length > 0 ? include : undefined,
|
||||
exclude: exclude.length > 0 ? exclude : undefined,
|
||||
syncEnabled: true,
|
||||
});
|
||||
console.log(`Added repo "${name}" (${strategy}) → ${repoPath}`);
|
||||
console.log(`${repos.length} repo(s) total. Run \`gbrain sync --all\` to index.`);
|
||||
} catch (e: unknown) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function reposRemove(args: string[]): void {
|
||||
if (args.length === 0) {
|
||||
console.error('Usage: gbrain repos remove <name>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const name = args[0];
|
||||
|
||||
try {
|
||||
const repos = removeRepoConfig(name);
|
||||
console.log(`Removed repo "${name}". ${repos.length} repo(s) remaining.`);
|
||||
} catch (e: unknown) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
+41
-7
@@ -41,6 +41,8 @@ export interface SyncOpts {
|
||||
skipFailed?: boolean;
|
||||
/** Bug 9 — re-attempt unacknowledged failures explicitly (CLI --retry-failed). */
|
||||
retryFailed?: boolean;
|
||||
/** Multi-repo: sync strategy override (markdown, code, auto). */
|
||||
strategy?: 'markdown' | 'code' | 'auto';
|
||||
}
|
||||
|
||||
function git(repoPath: string, ...args: string[]): string {
|
||||
@@ -127,16 +129,17 @@ export async function performSync(engine: BrainEngine, opts: SyncOpts): Promise<
|
||||
const diffOutput = git(repoPath, 'diff', '--name-status', '-M', `${lastCommit}..${headCommit}`);
|
||||
const manifest = buildSyncManifest(diffOutput);
|
||||
|
||||
// Filter to syncable files
|
||||
// Filter to syncable files (strategy-aware)
|
||||
const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined;
|
||||
const filtered: SyncManifest = {
|
||||
added: manifest.added.filter(p => isSyncable(p)),
|
||||
modified: manifest.modified.filter(p => isSyncable(p)),
|
||||
deleted: manifest.deleted.filter(p => isSyncable(p)),
|
||||
renamed: manifest.renamed.filter(r => isSyncable(r.to)),
|
||||
added: manifest.added.filter(p => isSyncable(p, syncOpts)),
|
||||
modified: manifest.modified.filter(p => isSyncable(p, syncOpts)),
|
||||
deleted: manifest.deleted.filter(p => isSyncable(p, syncOpts)),
|
||||
renamed: manifest.renamed.filter(r => isSyncable(r.to, syncOpts)),
|
||||
};
|
||||
|
||||
// Delete pages that became un-syncable (modified but filtered out)
|
||||
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p));
|
||||
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p, syncOpts));
|
||||
for (const path of unsyncableModified) {
|
||||
const slug = pathToSlug(path);
|
||||
try {
|
||||
@@ -481,8 +484,39 @@ export async function runSync(engine: BrainEngine, args: string[]) {
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
const skipFailed = args.includes('--skip-failed');
|
||||
const retryFailed = args.includes('--retry-failed');
|
||||
const syncAll = args.includes('--all');
|
||||
const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined;
|
||||
|
||||
const opts: SyncOpts = { repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed };
|
||||
// Multi-repo: --all syncs all configured repos
|
||||
if (syncAll) {
|
||||
const { loadRepoConfigs } = await import('../core/multi-repo.ts');
|
||||
const repos = loadRepoConfigs();
|
||||
if (repos.length === 0) {
|
||||
console.log('No repos configured. Use `gbrain repos add <path>` first.');
|
||||
return;
|
||||
}
|
||||
for (const repo of repos) {
|
||||
if (repo.syncEnabled === false) {
|
||||
console.log(`Skipping disabled repo: ${repo.name}`);
|
||||
continue;
|
||||
}
|
||||
console.log(`\n--- Syncing repo: ${repo.name} (${repo.strategy}) ---`);
|
||||
const repoOpts: SyncOpts = {
|
||||
repoPath: repo.path,
|
||||
dryRun, full, noPull, noEmbed, skipFailed, retryFailed,
|
||||
strategy: repo.strategy,
|
||||
};
|
||||
try {
|
||||
const result = await performSync(engine, repoOpts);
|
||||
printSyncResult(result);
|
||||
} catch (e: unknown) {
|
||||
console.error(`Error syncing ${repo.name}: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const opts: SyncOpts = { repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed, strategy: strategyArg };
|
||||
|
||||
// Bug 9 — --retry-failed: before running normal sync, clear acknowledgment
|
||||
// flags so the sync picks them up as fresh work. The actual re-attempt
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
/**
|
||||
* Code Chunker — Tree-Sitter-Based Semantic Code Splitting
|
||||
*
|
||||
* Uses web-tree-sitter (WASM) to parse code files into AST, then extracts
|
||||
* semantic units (functions, classes, types, exports) as chunks.
|
||||
*
|
||||
* Each chunk includes a structured header with language, file path, line range,
|
||||
* and symbol name — so embeddings capture both context and code content.
|
||||
*
|
||||
* Supports: TypeScript, TSX, JavaScript, Python, Ruby, Go.
|
||||
* Falls back to recursive text chunker for unsupported languages.
|
||||
*/
|
||||
|
||||
import { chunkText as recursiveChunk } from './recursive.ts';
|
||||
|
||||
// Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
|
||||
let Parser: typeof import('web-tree-sitter') | null = null;
|
||||
|
||||
async function getParser(): Promise<typeof import('web-tree-sitter')> {
|
||||
if (!Parser) {
|
||||
Parser = (await import('web-tree-sitter')).default || await import('web-tree-sitter');
|
||||
}
|
||||
return Parser;
|
||||
}
|
||||
|
||||
export type SupportedCodeLanguage = 'typescript' | 'tsx' | 'javascript' | 'python' | 'ruby' | 'go';
|
||||
|
||||
export interface CodeChunkMetadata {
|
||||
symbolName: string | null;
|
||||
symbolType: string;
|
||||
filePath: string;
|
||||
language: SupportedCodeLanguage;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
export interface CodeChunk {
|
||||
text: string;
|
||||
index: number;
|
||||
metadata: CodeChunkMetadata;
|
||||
}
|
||||
|
||||
export interface CodeChunkOptions {
|
||||
chunkSizeTokens?: number;
|
||||
largeChunkThresholdTokens?: number;
|
||||
fallbackChunkSizeWords?: number;
|
||||
fallbackOverlapWords?: number;
|
||||
}
|
||||
|
||||
const GRAMMAR_FILES: Record<SupportedCodeLanguage, string> = {
|
||||
typescript: 'tree-sitter-typescript.wasm',
|
||||
tsx: 'tree-sitter-tsx.wasm',
|
||||
javascript: 'tree-sitter-javascript.wasm',
|
||||
python: 'tree-sitter-python.wasm',
|
||||
ruby: 'tree-sitter-ruby.wasm',
|
||||
go: 'tree-sitter-go.wasm',
|
||||
};
|
||||
|
||||
const TOP_LEVEL_TYPES: Record<SupportedCodeLanguage, Set<string>> = {
|
||||
typescript: new Set([
|
||||
'function_declaration',
|
||||
'class_declaration',
|
||||
'abstract_class_declaration',
|
||||
'interface_declaration',
|
||||
'type_alias_declaration',
|
||||
'enum_declaration',
|
||||
'lexical_declaration',
|
||||
'variable_declaration',
|
||||
'export_statement',
|
||||
]),
|
||||
tsx: new Set([
|
||||
'function_declaration',
|
||||
'class_declaration',
|
||||
'interface_declaration',
|
||||
'type_alias_declaration',
|
||||
'enum_declaration',
|
||||
'lexical_declaration',
|
||||
'variable_declaration',
|
||||
'export_statement',
|
||||
]),
|
||||
javascript: new Set([
|
||||
'function_declaration',
|
||||
'class_declaration',
|
||||
'lexical_declaration',
|
||||
'variable_declaration',
|
||||
'export_statement',
|
||||
]),
|
||||
python: new Set([
|
||||
'function_definition',
|
||||
'class_definition',
|
||||
'import_statement',
|
||||
'import_from_statement',
|
||||
'assignment',
|
||||
]),
|
||||
ruby: new Set([
|
||||
'class',
|
||||
'module',
|
||||
'method',
|
||||
'singleton_method',
|
||||
'assignment',
|
||||
]),
|
||||
go: new Set([
|
||||
'function_declaration',
|
||||
'method_declaration',
|
||||
'type_declaration',
|
||||
'const_declaration',
|
||||
'var_declaration',
|
||||
'import_declaration',
|
||||
]),
|
||||
};
|
||||
|
||||
const BODY_NODE_TYPES = new Set([
|
||||
'statement_block',
|
||||
'block',
|
||||
'class_body',
|
||||
'module_body',
|
||||
'body_statement',
|
||||
'body',
|
||||
]);
|
||||
|
||||
let initDone = false;
|
||||
let initPromise: Promise<void> | null = null;
|
||||
const languageCache = new Map<SupportedCodeLanguage, any>();
|
||||
|
||||
// ---------- Public API ----------
|
||||
|
||||
export function detectCodeLanguage(filePath: string): SupportedCodeLanguage | null {
|
||||
const lower = filePath.toLowerCase();
|
||||
if (lower.endsWith('.tsx')) return 'tsx';
|
||||
if (lower.endsWith('.ts')) return 'typescript';
|
||||
if (lower.endsWith('.js') || lower.endsWith('.jsx') || lower.endsWith('.mjs') || lower.endsWith('.cjs')) return 'javascript';
|
||||
if (lower.endsWith('.py')) return 'python';
|
||||
if (lower.endsWith('.rb')) return 'ruby';
|
||||
if (lower.endsWith('.go')) return 'go';
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function chunkCodeText(
|
||||
source: string,
|
||||
filePath: string,
|
||||
opts: CodeChunkOptions = {},
|
||||
): Promise<CodeChunk[]> {
|
||||
const language = detectCodeLanguage(filePath);
|
||||
if (!language) {
|
||||
return fallbackChunks(source, filePath, 'javascript', opts);
|
||||
}
|
||||
|
||||
if (!source.trim()) return [];
|
||||
|
||||
const largeThreshold = opts.largeChunkThresholdTokens ?? 1000;
|
||||
const chunkTarget = opts.chunkSizeTokens ?? 300;
|
||||
|
||||
try {
|
||||
await ensureInit();
|
||||
const P = await getParser();
|
||||
const parser = new (P as any)();
|
||||
const grammar = await loadLanguage(language);
|
||||
parser.setLanguage(grammar);
|
||||
|
||||
const tree = parser.parse(source);
|
||||
if (!tree) {
|
||||
parser.delete();
|
||||
return fallbackChunks(source, filePath, language, opts);
|
||||
}
|
||||
|
||||
const root = tree.rootNode;
|
||||
const topLevelTypes = TOP_LEVEL_TYPES[language];
|
||||
const semanticNodes = root.namedChildren.filter((n: any) => topLevelTypes.has(n.type));
|
||||
|
||||
if (semanticNodes.length === 0) {
|
||||
tree.delete();
|
||||
parser.delete();
|
||||
return fallbackChunks(source, filePath, language, opts);
|
||||
}
|
||||
|
||||
const chunks: CodeChunk[] = [];
|
||||
for (const node of semanticNodes) {
|
||||
const symbolName = extractSymbolName(node);
|
||||
const symbolType = normalizeSymbolType(node.type);
|
||||
const nodeText = source.slice(node.startIndex, node.endIndex).trim();
|
||||
if (!nodeText) continue;
|
||||
|
||||
if (estimateTokens(nodeText) <= largeThreshold) {
|
||||
chunks.push(buildChunk({
|
||||
body: nodeText, filePath, language, symbolName, symbolType,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
index: chunks.length,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Split very large nodes at nested block boundaries
|
||||
const subRanges = splitLargeNode(node, source, chunkTarget);
|
||||
if (subRanges.length === 0) {
|
||||
chunks.push(buildChunk({
|
||||
body: nodeText, filePath, language, symbolName, symbolType,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
index: chunks.length,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const range of subRanges) {
|
||||
const body = source.slice(range.startIndex, range.endIndex).trim();
|
||||
if (!body) continue;
|
||||
chunks.push(buildChunk({
|
||||
body, filePath, language, symbolName, symbolType,
|
||||
startLine: range.startLine, endLine: range.endLine,
|
||||
index: chunks.length,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
tree.delete();
|
||||
parser.delete();
|
||||
|
||||
return chunks.length > 0 ? chunks : fallbackChunks(source, filePath, language, opts);
|
||||
} catch {
|
||||
return fallbackChunks(source, filePath, language, opts);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Internals ----------
|
||||
|
||||
function fallbackChunks(
|
||||
source: string,
|
||||
filePath: string,
|
||||
language: SupportedCodeLanguage,
|
||||
opts: CodeChunkOptions,
|
||||
): CodeChunk[] {
|
||||
const size = opts.fallbackChunkSizeWords ?? 300;
|
||||
const overlap = opts.fallbackOverlapWords ?? 50;
|
||||
return recursiveChunk(source, { chunkSize: size, chunkOverlap: overlap }).map((chunk, index) =>
|
||||
buildChunk({
|
||||
body: chunk.text, filePath, language,
|
||||
symbolName: null, symbolType: 'module',
|
||||
startLine: 1, endLine: countLines(chunk.text),
|
||||
index,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function buildChunk(input: {
|
||||
body: string;
|
||||
filePath: string;
|
||||
language: SupportedCodeLanguage;
|
||||
symbolName: string | null;
|
||||
symbolType: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
index: number;
|
||||
}): CodeChunk {
|
||||
const symbol = input.symbolName ? `${input.symbolType} ${input.symbolName}` : input.symbolType;
|
||||
const header = `[${displayLang(input.language)}] ${input.filePath}:${input.startLine}-${input.endLine} ${symbol}`;
|
||||
return {
|
||||
index: input.index,
|
||||
text: `${header}\n\n${input.body}`,
|
||||
metadata: {
|
||||
symbolName: input.symbolName,
|
||||
symbolType: input.symbolType,
|
||||
filePath: input.filePath,
|
||||
language: input.language,
|
||||
startLine: input.startLine,
|
||||
endLine: input.endLine,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface SplitRange {
|
||||
startIndex: number;
|
||||
endIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
function splitLargeNode(node: any, source: string, chunkTarget: number): SplitRange[] {
|
||||
const body =
|
||||
node.childForFieldName('body') ||
|
||||
node.namedChildren.find((c: any) => BODY_NODE_TYPES.has(c.type)) ||
|
||||
null;
|
||||
|
||||
if (!body || body.namedChildren.length < 2) return [];
|
||||
|
||||
const children = body.namedChildren.filter((c: any) => !c.isExtra);
|
||||
if (children.length < 2) return [];
|
||||
|
||||
const ranges: SplitRange[] = [];
|
||||
let curStart = children[0].startIndex;
|
||||
let curStartLine = children[0].startPosition.row + 1;
|
||||
let curEnd = children[0].endIndex;
|
||||
let curEndLine = children[0].endPosition.row + 1;
|
||||
let curTokens = estimateTokens(source.slice(curStart, curEnd));
|
||||
|
||||
for (let i = 1; i < children.length; i++) {
|
||||
const child = children[i];
|
||||
const childTokens = estimateTokens(source.slice(child.startIndex, child.endIndex));
|
||||
|
||||
if (curTokens + childTokens > Math.ceil(chunkTarget * 1.5)) {
|
||||
ranges.push({ startIndex: curStart, endIndex: curEnd, startLine: curStartLine, endLine: curEndLine });
|
||||
curStart = child.startIndex;
|
||||
curStartLine = child.startPosition.row + 1;
|
||||
curEnd = child.endIndex;
|
||||
curEndLine = child.endPosition.row + 1;
|
||||
curTokens = childTokens;
|
||||
} else {
|
||||
curEnd = child.endIndex;
|
||||
curEndLine = child.endPosition.row + 1;
|
||||
curTokens += childTokens;
|
||||
}
|
||||
}
|
||||
ranges.push({ startIndex: curStart, endIndex: curEnd, startLine: curStartLine, endLine: curEndLine });
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function extractSymbolName(node: any): string | null {
|
||||
const directName = node.childForFieldName('name');
|
||||
if (directName?.text?.trim()) return sanitize(directName.text);
|
||||
|
||||
const declaration = node.childForFieldName('declaration');
|
||||
if (declaration) {
|
||||
const nested = extractSymbolName(declaration);
|
||||
if (nested) return nested;
|
||||
}
|
||||
|
||||
for (const child of node.namedChildren) {
|
||||
if (child.type.endsWith('identifier') || child.type === 'constant') {
|
||||
const v = sanitize(child.text);
|
||||
if (v) return v;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeSymbolType(type: string): string {
|
||||
if (type.includes('function') || type === 'method' || type === 'singleton_method') return 'function';
|
||||
if (type.includes('class')) return 'class';
|
||||
if (type.includes('interface')) return 'interface';
|
||||
if (type.includes('type_alias')) return 'type';
|
||||
if (type.includes('enum')) return 'enum';
|
||||
if (type.includes('module')) return 'module';
|
||||
if (type.includes('import')) return 'import';
|
||||
return type.replace(/_/g, ' ');
|
||||
}
|
||||
|
||||
function sanitize(name: string): string {
|
||||
return name.replace(/[\n\r\t]+/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.max(1, Math.ceil(text.length / 4));
|
||||
}
|
||||
|
||||
function displayLang(lang: SupportedCodeLanguage): string {
|
||||
const map: Record<SupportedCodeLanguage, string> = {
|
||||
typescript: 'TypeScript', tsx: 'TSX', javascript: 'JavaScript',
|
||||
python: 'Python', ruby: 'Ruby', go: 'Go',
|
||||
};
|
||||
return map[lang];
|
||||
}
|
||||
|
||||
function countLines(text: string): number {
|
||||
return text ? text.split('\n').length : 0;
|
||||
}
|
||||
|
||||
// ---------- Tree-sitter init ----------
|
||||
|
||||
async function ensureInit(): Promise<void> {
|
||||
if (initDone) return;
|
||||
if (!initPromise) {
|
||||
initPromise = (async () => {
|
||||
const P = await getParser();
|
||||
// v0.22.x: init takes locateFile for the WASM module
|
||||
const wasmPath = new URL('../../../node_modules/web-tree-sitter/tree-sitter.wasm', import.meta.url);
|
||||
let resolved: string;
|
||||
try {
|
||||
const { fileURLToPath } = await import('url');
|
||||
resolved = fileURLToPath(wasmPath);
|
||||
} catch {
|
||||
resolved = wasmPath.pathname;
|
||||
}
|
||||
await (P as any).init({ locateFile: () => resolved });
|
||||
initDone = true;
|
||||
})();
|
||||
}
|
||||
await initPromise;
|
||||
}
|
||||
|
||||
async function loadLanguage(language: SupportedCodeLanguage): Promise<any> {
|
||||
if (languageCache.has(language)) return languageCache.get(language);
|
||||
|
||||
const P = await getParser();
|
||||
const grammarUrl = new URL(
|
||||
`../../../node_modules/tree-sitter-wasms/out/${GRAMMAR_FILES[language]}`,
|
||||
import.meta.url,
|
||||
);
|
||||
let resolved: string;
|
||||
try {
|
||||
const { fileURLToPath } = await import('url');
|
||||
resolved = fileURLToPath(grammarUrl);
|
||||
} catch {
|
||||
resolved = grammarUrl.pathname;
|
||||
}
|
||||
const lang = await (P as any).Language.load(resolved);
|
||||
languageCache.set(language, lang);
|
||||
return lang;
|
||||
}
|
||||
@@ -27,6 +27,14 @@ export interface GBrainConfig {
|
||||
engine: 'postgres' | 'pglite';
|
||||
database_url?: string;
|
||||
database_path?: string;
|
||||
repos?: Array<{
|
||||
path: string;
|
||||
name: string;
|
||||
strategy: 'markdown' | 'code' | 'auto';
|
||||
include?: string[];
|
||||
exclude?: string[];
|
||||
syncEnabled?: boolean;
|
||||
}>;
|
||||
openai_api_key?: string;
|
||||
anthropic_api_key?: string;
|
||||
/**
|
||||
|
||||
+86
-1
@@ -1,10 +1,12 @@
|
||||
import { readFileSync, statSync, lstatSync } from 'fs';
|
||||
import { basename } from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { parseMarkdown } from './markdown.ts';
|
||||
import { chunkText } from './chunkers/recursive.ts';
|
||||
import { chunkCodeText, detectCodeLanguage } from './chunkers/code.ts';
|
||||
import { embedBatch } from './embedding.ts';
|
||||
import { slugifyPath } from './sync.ts';
|
||||
import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts';
|
||||
import type { ChunkInput, PageType } from './types.ts';
|
||||
|
||||
/**
|
||||
@@ -184,6 +186,12 @@ export async function importFromFile(
|
||||
}
|
||||
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
|
||||
// Route code files through the code import path
|
||||
if (isCodeFilePath(relativePath)) {
|
||||
return importCodeFile(engine, relativePath, content, opts);
|
||||
}
|
||||
|
||||
const parsed = parseMarkdown(content, relativePath);
|
||||
|
||||
// Enforce path-authoritative slug. parseMarkdown prefers frontmatter.slug over
|
||||
@@ -206,6 +214,83 @@ export async function importFromFile(
|
||||
return importFromContent(engine, expectedSlug, content, opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a code file. Bypasses markdown parsing entirely.
|
||||
* Uses tree-sitter code chunker for semantic splitting.
|
||||
* Page type is 'code', slug includes file extension.
|
||||
*/
|
||||
export async function importCodeFile(
|
||||
engine: BrainEngine,
|
||||
relativePath: string,
|
||||
content: string,
|
||||
opts: { noEmbed?: boolean } = {},
|
||||
): Promise<ImportResult> {
|
||||
const slug = slugifyCodePath(relativePath);
|
||||
const lang = detectCodeLanguage(relativePath) || 'unknown';
|
||||
const title = `${relativePath} (${lang})`;
|
||||
|
||||
const byteLength = Buffer.byteLength(content, 'utf-8');
|
||||
if (byteLength > MAX_FILE_SIZE) {
|
||||
return { slug, status: 'skipped', chunks: 0, error: `Code file too large (${byteLength} bytes)` };
|
||||
}
|
||||
|
||||
// Hash for idempotency
|
||||
const hash = createHash('sha256')
|
||||
.update(JSON.stringify({ title, type: 'code', content, lang }))
|
||||
.digest('hex');
|
||||
|
||||
const existing = await engine.getPage(slug);
|
||||
if (existing?.content_hash === hash) {
|
||||
return { slug, status: 'skipped', chunks: 0 };
|
||||
}
|
||||
|
||||
// Chunk via tree-sitter code chunker
|
||||
const codeChunks = await chunkCodeText(content, relativePath);
|
||||
const chunks: ChunkInput[] = codeChunks.map((c, i) => ({
|
||||
chunk_index: i,
|
||||
chunk_text: c.text,
|
||||
chunk_source: 'compiled_truth' as const,
|
||||
}));
|
||||
|
||||
// Embed
|
||||
if (!opts.noEmbed && chunks.length > 0) {
|
||||
try {
|
||||
const embeddings = await embedBatch(chunks.map(c => c.chunk_text));
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
chunks[i].embedding = embeddings[i];
|
||||
chunks[i].token_count = Math.ceil(chunks[i].chunk_text.length / 4);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.warn(`[gbrain] embedding failed for code file ${slug}: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Store
|
||||
await engine.transaction(async (tx) => {
|
||||
if (existing) await tx.createVersion(slug);
|
||||
|
||||
await tx.putPage(slug, {
|
||||
type: 'code' as PageType,
|
||||
title,
|
||||
compiled_truth: content,
|
||||
timeline: '',
|
||||
frontmatter: { language: lang, file: relativePath },
|
||||
content_hash: hash,
|
||||
});
|
||||
|
||||
await tx.addTag(slug, 'code');
|
||||
await tx.addTag(slug, lang);
|
||||
|
||||
if (chunks.length > 0) {
|
||||
await tx.upsertChunks(slug, chunks);
|
||||
} else {
|
||||
await tx.deleteChunks(slug);
|
||||
}
|
||||
});
|
||||
|
||||
return { slug, status: 'imported', chunks: chunks.length };
|
||||
}
|
||||
|
||||
// Backward compat
|
||||
export const importFile = importFromFile;
|
||||
export type ImportFileResult = ImportResult;
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
||||
import { resolve } from 'path';
|
||||
import { configDir, configPath } from './config.ts';
|
||||
|
||||
export type RepoStrategy = 'markdown' | 'code' | 'auto';
|
||||
|
||||
export interface RepoConfig {
|
||||
path: string;
|
||||
name: string;
|
||||
strategy: RepoStrategy;
|
||||
include?: string[];
|
||||
exclude?: string[];
|
||||
syncEnabled?: boolean;
|
||||
}
|
||||
|
||||
interface RawConfig {
|
||||
[key: string]: unknown;
|
||||
repos?: unknown;
|
||||
}
|
||||
|
||||
function readRawConfig(): RawConfig {
|
||||
try {
|
||||
const raw = readFileSync(configPath(), 'utf-8');
|
||||
return JSON.parse(raw) as RawConfig;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function writeRawConfig(config: RawConfig): void {
|
||||
mkdirSync(configDir(), { recursive: true });
|
||||
writeFileSync(configPath(), JSON.stringify(config, null, 2) + '\n', { mode: 0o600 });
|
||||
try {
|
||||
chmodSync(configPath(), 0o600);
|
||||
} catch {
|
||||
// chmod may fail on some platforms
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeRepoName(repoPath: string): string {
|
||||
const normalized = repoPath.replace(/\\/g, '/').replace(/\/+$/g, '');
|
||||
const parts = normalized.split('/').filter(Boolean);
|
||||
return parts[parts.length - 1] || 'repo';
|
||||
}
|
||||
|
||||
export function loadRepoConfigs(): RepoConfig[] {
|
||||
if (!existsSync(configPath())) return [];
|
||||
|
||||
const parsed = readRawConfig();
|
||||
if (!Array.isArray(parsed.repos)) return [];
|
||||
|
||||
const repos: RepoConfig[] = [];
|
||||
for (const candidate of parsed.repos) {
|
||||
if (!candidate || typeof candidate !== 'object') continue;
|
||||
const row = candidate as Record<string, unknown>;
|
||||
if (typeof row.path !== 'string' || typeof row.name !== 'string') continue;
|
||||
|
||||
const strategy = row.strategy;
|
||||
const normalizedStrategy: RepoStrategy = strategy === 'code' || strategy === 'auto' ? strategy : 'markdown';
|
||||
|
||||
repos.push({
|
||||
path: resolve(row.path),
|
||||
name: row.name,
|
||||
strategy: normalizedStrategy,
|
||||
include: Array.isArray(row.include) ? row.include.map(String) : undefined,
|
||||
exclude: Array.isArray(row.exclude) ? row.exclude.map(String) : undefined,
|
||||
syncEnabled: typeof row.syncEnabled === 'boolean' ? row.syncEnabled : true,
|
||||
});
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
export function saveRepoConfigs(repos: RepoConfig[]): void {
|
||||
const parsed = readRawConfig();
|
||||
parsed.repos = repos.map((repo) => ({
|
||||
path: resolve(repo.path),
|
||||
name: repo.name,
|
||||
strategy: repo.strategy,
|
||||
...(repo.include && repo.include.length > 0 ? { include: repo.include } : {}),
|
||||
...(repo.exclude && repo.exclude.length > 0 ? { exclude: repo.exclude } : {}),
|
||||
...(repo.syncEnabled === false ? { syncEnabled: false } : {}),
|
||||
}));
|
||||
writeRawConfig(parsed);
|
||||
}
|
||||
|
||||
export function addRepoConfig(repo: RepoConfig): RepoConfig[] {
|
||||
const repos = loadRepoConfigs();
|
||||
const normalized: RepoConfig = {
|
||||
...repo,
|
||||
path: resolve(repo.path),
|
||||
name: repo.name || normalizeRepoName(repo.path),
|
||||
strategy: repo.strategy || 'auto',
|
||||
syncEnabled: repo.syncEnabled ?? true,
|
||||
};
|
||||
|
||||
const nameTaken = repos.find((r) => r.name === normalized.name);
|
||||
if (nameTaken) {
|
||||
throw new Error(`Repo name already exists: ${normalized.name}`);
|
||||
}
|
||||
|
||||
const pathTaken = repos.find((r) => resolve(r.path) === normalized.path);
|
||||
if (pathTaken) {
|
||||
throw new Error(`Repo path already configured: ${normalized.path}`);
|
||||
}
|
||||
|
||||
const updated = [...repos, normalized];
|
||||
saveRepoConfigs(updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
export function removeRepoConfig(name: string): RepoConfig[] {
|
||||
const repos = loadRepoConfigs();
|
||||
const next = repos.filter((r) => r.name !== name);
|
||||
if (next.length === repos.length) {
|
||||
throw new Error(`Repo not found: ${name}`);
|
||||
}
|
||||
saveRepoConfigs(next);
|
||||
return next;
|
||||
}
|
||||
+85
-5
@@ -24,6 +24,16 @@ export interface RawManifestEntry {
|
||||
oldPath?: string;
|
||||
}
|
||||
|
||||
export type SyncStrategy = 'markdown' | 'code' | 'auto';
|
||||
|
||||
interface SyncableOptions {
|
||||
strategy?: SyncStrategy;
|
||||
include?: string[];
|
||||
exclude?: string[];
|
||||
}
|
||||
|
||||
const CODE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py', '.rb', '.go']);
|
||||
|
||||
/**
|
||||
* Parse the output of `git diff --name-status -M LAST..HEAD` into structured entries.
|
||||
*
|
||||
@@ -72,12 +82,60 @@ export function buildSyncManifest(gitDiffOutput: string): SyncManifest {
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function isCodeFilePath(path: string): boolean {
|
||||
const lower = path.toLowerCase();
|
||||
for (const ext of CODE_EXTENSIONS) {
|
||||
if (lower.endsWith(ext)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isMarkdownFilePath(path: string): boolean {
|
||||
return path.endsWith('.md') || path.endsWith('.mdx');
|
||||
}
|
||||
|
||||
function isAllowedByStrategy(path: string, strategy: SyncStrategy): boolean {
|
||||
if (strategy === 'markdown') return isMarkdownFilePath(path);
|
||||
if (strategy === 'code') return isCodeFilePath(path);
|
||||
return isMarkdownFilePath(path) || isCodeFilePath(path);
|
||||
}
|
||||
|
||||
function globToRegex(pattern: string): RegExp {
|
||||
let regex = '^';
|
||||
for (let i = 0; i < pattern.length; i++) {
|
||||
const ch = pattern[i];
|
||||
if (ch === '*') {
|
||||
const next = pattern[i + 1];
|
||||
if (next === '*') {
|
||||
regex += '.*';
|
||||
i++;
|
||||
} else {
|
||||
regex += '[^/]*';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === '?') { regex += '.'; continue; }
|
||||
if ('\\.[]{}()+-^$|'.includes(ch)) { regex += `\\${ch}`; continue; }
|
||||
regex += ch;
|
||||
}
|
||||
regex += '$';
|
||||
return new RegExp(regex);
|
||||
}
|
||||
|
||||
function matchesAnyGlob(path: string, patterns?: string[]): boolean {
|
||||
if (!patterns || patterns.length === 0) return false;
|
||||
const normalized = path.replace(/\\/g, '/');
|
||||
return patterns.some((pattern) => globToRegex(pattern).test(normalized));
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter a file path to determine if it should be synced to GBrain.
|
||||
* Strategy-aware: 'markdown' (default) = .md/.mdx only, 'code' = code files only, 'auto' = both.
|
||||
*/
|
||||
export function isSyncable(path: string): boolean {
|
||||
// Must be .md or .mdx
|
||||
if (!path.endsWith('.md') && !path.endsWith('.mdx')) return false;
|
||||
export function isSyncable(path: string, opts: SyncableOptions = {}): boolean {
|
||||
const strategy = opts.strategy || 'markdown';
|
||||
|
||||
if (!isAllowedByStrategy(path, strategy)) return false;
|
||||
|
||||
// Skip hidden directories
|
||||
if (path.split('/').some(p => p.startsWith('.'))) return false;
|
||||
@@ -93,6 +151,9 @@ export function isSyncable(path: string): boolean {
|
||||
// Skip ops/ directory
|
||||
if (path.startsWith('ops/')) return false;
|
||||
|
||||
if (opts.include && opts.include.length > 0 && !matchesAnyGlob(path, opts.include)) return false;
|
||||
if (opts.exclude && opts.exclude.length > 0 && matchesAnyGlob(path, opts.exclude)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -125,11 +186,30 @@ export function slugifyPath(filePath: string): string {
|
||||
return path.split('/').map(slugifySegment).filter(Boolean).join('/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Slugify a code file path: flatten into a single slug segment with dots → hyphens.
|
||||
* e.g. 'src/core/chunkers/code.ts' → 'src-core-chunkers-code-ts'
|
||||
*/
|
||||
export function slugifyCodePath(filePath: string): string {
|
||||
let path = filePath.replace(/\\/g, '/');
|
||||
path = path.replace(/^\.?\//, '');
|
||||
return path
|
||||
.split('/')
|
||||
.map(segment => slugifySegment(segment.replace(/\./g, '-')))
|
||||
.filter(Boolean)
|
||||
.join('-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a repo-relative file path to a GBrain page slug.
|
||||
*/
|
||||
export function pathToSlug(filePath: string, repoPrefix?: string): string {
|
||||
let slug = slugifyPath(filePath);
|
||||
export function pathToSlug(
|
||||
filePath: string,
|
||||
repoPrefix?: string,
|
||||
options: { pageKind?: 'markdown' | 'code' } = {},
|
||||
): string {
|
||||
const pageKind = options.pageKind || 'markdown';
|
||||
let slug = pageKind === 'code' ? slugifyCodePath(filePath) : slugifyPath(filePath);
|
||||
if (repoPrefix) slug = `${repoPrefix}/${slug}`;
|
||||
return slug.toLowerCase();
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
// Page types
|
||||
export type PageType = 'person' | 'company' | 'deal' | 'yc' | 'civic' | 'project' | 'concept' | 'source' | 'media' | 'writing' | 'analysis' | 'guide' | 'hardware' | 'architecture' | 'meeting' | 'note';
|
||||
export type PageType = 'person' | 'company' | 'deal' | 'yc' | 'civic' | 'project' | 'concept' | 'source' | 'media' | 'writing' | 'analysis' | 'guide' | 'hardware' | 'architecture' | 'meeting' | 'note' | 'code';
|
||||
|
||||
export interface Page {
|
||||
id: number;
|
||||
|
||||
Reference in New Issue
Block a user