diff --git a/src/commands/import.ts b/src/commands/import.ts index 704970cd2..8d50341ee 100644 --- a/src/commands/import.ts +++ b/src/commands/import.ts @@ -59,6 +59,11 @@ export async function runImport( * Threaded by performFullSync for `gbrain sync --exclude`. */ exclude?: string[]; + /** + * Opt out of the git-visible fast path and walk the filesystem directly, + * so markdown/code files matched by .gitignore can still be imported. + */ + includeGitignored?: boolean; /** * #753/#774 monorepo subdir-source support: when set, slugs and * `source_path` are computed relative to this root (the git repo root) @@ -71,6 +76,7 @@ export async function runImport( const noEmbed = args.includes('--no-embed'); const fresh = args.includes('--fresh'); const jsonOutput = args.includes('--json'); + const includeGitignored = args.includes('--include-gitignored') || opts.includeGitignored === true; // T7 (D9): refuse cleanly when init persisted the deferred-setup sentinel, // unless the user is explicitly skipping embedding via `--no-embed` (in @@ -185,7 +191,7 @@ export async function runImport( const dirArg = args.find((a, i) => !a.startsWith('--') && !flagValues.has(i)); if (!dirArg) { - console.error('Usage: gbrain import [--no-embed] [--workers N] [--fresh] [--source-id ] [--json]'); + console.error('Usage: gbrain import [--no-embed] [--workers N] [--fresh] [--source-id ] [--include-gitignored] [--json]'); process.exit(1); } // #1728: capture the import target ONCE as an absolute real path. Every @@ -209,7 +215,7 @@ export async function runImport( const strategy: SyncStrategy = opts.strategy ?? 'markdown'; const _walkT0 = Date.now(); console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`); - let allFiles = collectSyncableFiles(dir, { strategy }); + let allFiles = collectSyncableFiles(dir, { strategy, includeGitignored }); console.error( `[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`, ); @@ -545,6 +551,7 @@ function resolveMaxWalkDepth(): number { interface CollectOpts { strategy?: SyncStrategy; + includeGitignored?: boolean; } /** @@ -675,8 +682,10 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin // vendored data/fixtures). `--cached --others --exclude-standard` = tracked // PLUS untracked-not-ignored, so uncommitted source is still indexed. Non-git // dirs (or git unavailable) fall through to the FS walk below. - const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn); - if (gitFiles) return gitFiles; + if (!opts.includeGitignored) { + const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn); + if (gitFiles) return gitFiles; + } const maxDepth = resolveMaxWalkDepth(); const visitedInodes = new Map(); diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 8844a1d58..1c28b2280 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -239,11 +239,12 @@ export interface SyncResult { export function estimateSourceTreeTokens( localPath: string, strategy: 'markdown' | 'code' | 'auto', + opts: { includeGitignored?: boolean } = {}, ): { tokens: number; files: number } { let tokens = 0; let files = 0; try { - const fileList = collectSyncableFiles(localPath, { strategy }); + const fileList = collectSyncableFiles(localPath, { strategy, includeGitignored: opts.includeGitignored }); for (const fullPath of fileList) { try { const stat = statSync(fullPath); @@ -376,6 +377,7 @@ export function estimateInlineNewTokens( chunker_version: string | null; }>, currentChunkerVersion: string, + opts: { forceFullTree?: boolean } = {}, ): InlineEstimate { let tokens = 0; let changedSources = 0; @@ -398,6 +400,14 @@ export function estimateInlineNewTokens( const strategy = cfg.strategy ?? 'markdown'; const localPath = src.local_path; + if (opts.forceFullTree) { + tokens += estimateSourceTreeTokens(localPath, strategy, { includeGitignored: true }).tokens; + changedSources++; + hadCeiling = true; + ceilingReasons.push('include_gitignored'); + continue; + } + // Rung 2: chunker drift forces a full re-chunk → full re-embed. CEILING. if (src.chunker_version !== currentChunkerVersion) { ceiling(localPath, strategy, 'chunker_drift'); @@ -542,6 +552,7 @@ interface CostGateContext { jsonOut: boolean; yesFlag: boolean; full: boolean; + includeGitignored?: boolean; /** Message prefix ('sync --all' | 'sync'). */ label: string; } @@ -626,7 +637,9 @@ async function runInlineCostGate( } // ── Inline path ─────────────────────────────────────────────── - const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION)); + const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION), { + forceFullTree: ctx.includeGitignored === true, + }); // D7A: `--full` runs `performFullSync` → `runEmbedCore({stale:true})`, which // sweeps the pre-existing stale backlog INLINE on top of the delta. Price it. const costUsd = estimateEmbeddingCostUsd(inline.tokens) + (full ? staleCostUsd : 0); @@ -764,6 +777,11 @@ export interface SyncOpts { * matching the #1433 metafile posture). */ exclude?: string[]; + /** + * Include files matched by .gitignore. Git cannot report untracked ignored + * changes in diffs, so sync uses the full filesystem walker when this is set. + */ + includeGitignored?: boolean; /** * Number of parallel workers for the import phase. When > 1, each worker * gets its own small Postgres connection pool and files are dispatched via @@ -2170,6 +2188,14 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise 0) { allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude)); } @@ -3603,6 +3632,7 @@ async function performFullSync( const { runImport } = await import('./import.ts'); const importArgs = [syncScopeRoot]; if (opts.noEmbed) importArgs.push('--no-embed'); + if (opts.includeGitignored) importArgs.push('--include-gitignored'); if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency)); // v0.31.2: thread strategy through so code-strategy first sync // actually enumerates code files (closes bug 1). @@ -3616,6 +3646,7 @@ async function performFullSync( strategy: opts.strategy, sourceId: opts.sourceId, exclude: opts.exclude, + includeGitignored: opts.includeGitignored, slugRoot, // issue #1939: performFullSync owns the failure ledger + bookmark via the // shared gate below; don't let runImport double-record or write its own. @@ -3728,7 +3759,10 @@ async function performFullSync( // #774: scoped syncs store git-root-relative source_paths (slugRoot), so // relativize the walk to the same base — otherwise every page mismatches // and the mass-delete valve trips on a perfectly healthy scoped source. - const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' }) + const currentFiles = collectSyncableFiles(syncScopeRoot, { + strategy: opts.strategy ?? 'markdown', + includeGitignored: opts.includeGitignored, + }) .map(abs => relative(slugRoot ?? syncScopeRoot, abs)); const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>( `SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`, @@ -4109,6 +4143,9 @@ Options: subdirectory directly as --repo also works. --exclude Exclude files matching the glob from sync (repeatable; matched against the scope-relative path). + --include-gitignored Include otherwise-syncable files matched by .gitignore. + Forces a full filesystem walk so periodic syncs see + ignored untracked content. --dry-run Show what would be synced without writing. --skip-failed Acknowledge previously-recorded sync failures so the bookmark can advance past unparseable files. @@ -4159,6 +4196,7 @@ See also: const skipFailed = args.includes('--skip-failed'); const retryFailed = args.includes('--retry-failed'); const noSchemaPack = args.includes('--no-schema-pack'); // v0.41.37.0 #1569 + const includeGitignored = args.includes('--include-gitignored'); const syncAll = args.includes('--all'); const jsonOut = args.includes('--json'); const yesFlag = args.includes('--yes'); @@ -4415,7 +4453,7 @@ See also: if (!noEmbed) { const mode = willEmbedSynchronously({ v2Enabled, serialFlag, noEmbed }); const gate = await runInlineCostGate(engine, { - sources, mode, dryRun, jsonOut, yesFlag, full, label: 'sync --all', + sources, mode, dryRun, jsonOut, yesFlag, full, includeGitignored, label: 'sync --all', }); if (gate.action === 'stop') return; autoDeferEmbeds = gate.autoDeferEmbeds; @@ -4513,6 +4551,7 @@ See also: noEmbed: effectiveNoEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, + includeGitignored, sourceId: src.id, strategy: cfg.strategy, concurrency, @@ -4737,7 +4776,7 @@ See also: const singleSourceInterrupt = new AbortController(); const onSingleSourceSigint = () => { try { singleSourceInterrupt.abort(new Error('SIGINT')); } catch { /* */ } }; const opts: SyncOpts = { - repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId, + repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, includeGitignored, sourceId, strategy: strategyArg, concurrency, srcSubpath, exclude: excludePatterns.length > 0 ? excludePatterns : undefined, @@ -4766,7 +4805,7 @@ See also: chunker_version: gateRows[0].chunker_version, }]; const gate = await runInlineCostGate(engine, { - sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, label: 'sync', + sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, includeGitignored, label: 'sync', }); if (gate.action === 'stop') return; if (gate.autoDeferEmbeds) { @@ -4976,6 +5015,7 @@ export async function syncOneSource( noSchemaPack?: boolean; /** v0.42.7 #1696: propagate --no-extract into every per-source sync. */ noExtract?: boolean; + includeGitignored?: boolean; }, ): Promise<{ result: SyncResult; log: string }> { const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' }; @@ -4990,6 +5030,7 @@ export async function syncOneSource( skipFailed: shared.skipFailed, retryFailed: shared.retryFailed, noSchemaPack: shared.noSchemaPack, + includeGitignored: shared.includeGitignored, sourceId: src.id, strategy: cfg.strategy, concurrency: shared.concurrency, diff --git a/test/import-git-fastpath-prune.test.ts b/test/import-git-fastpath-prune.test.ts index f591646e2..8a9238215 100644 --- a/test/import-git-fastpath-prune.test.ts +++ b/test/import-git-fastpath-prune.test.ts @@ -87,4 +87,27 @@ describe('#2607 — git fast path excludes what incremental sync excludes', () = } expect(files.length).toBeGreaterThan(0); }); + + test('--include-gitignored falls back to filesystem walk for ignored content', () => { + const ignoredRepo = mkdtempSync(join(tmpdir(), 'gbrain-include-ignored-')); + try { + execSync('git init', { cwd: ignoredRepo, stdio: 'pipe' }); + writeFileSync(join(ignoredRepo, '.gitignore'), 'Meetings/\n'); + writeFileSync(join(ignoredRepo, 'notes.md'), '---\ntitle: Notes\n---\nbody\n'); + mkdirSync(join(ignoredRepo, 'Meetings'), { recursive: true }); + writeFileSync(join(ignoredRepo, 'Meetings/weekly.md'), '---\ntitle: Weekly\n---\nbody\n'); + + const toRel = (files: string[]) => files.map((f) => relative(ignoredRepo, f)); + const defaultFiles = toRel(collectSyncableFiles(ignoredRepo, { strategy: 'markdown' })); + const includeIgnored = toRel(collectSyncableFiles(ignoredRepo, { + strategy: 'markdown', + includeGitignored: true, + })); + + expect(defaultFiles).not.toContain('Meetings/weekly.md'); + expect(includeIgnored).toContain('Meetings/weekly.md'); + } finally { + rmSync(ignoredRepo, { recursive: true, force: true }); + } + }); }); diff --git a/test/sync.test.ts b/test/sync.test.ts index 204408f02..c7835b35b 100644 --- a/test/sync.test.ts +++ b/test/sync.test.ts @@ -464,6 +464,59 @@ describe('performSync dry-run never writes', () => { expect(typeof result.embedded).toBe('number'); }); + test('--include-gitignored imports ignored files even when git HEAD is unchanged', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const first = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + noExtract: true, + }); + expect(first.status).toBe('first_sync'); + + writeFileSync(join(repoPath, '.gitignore'), 'Meetings/\n'); + execSync('git add .gitignore && git commit -m "ignore generated meetings"', { cwd: repoPath, stdio: 'pipe' }); + const checkpoint = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + noExtract: true, + }); + expect(checkpoint.status).toBe('up_to_date'); + + mkdirSync(join(repoPath, 'Meetings'), { recursive: true }); + writeFileSync(join(repoPath, 'Meetings/weekly.md'), [ + '---', + 'type: meeting', + 'title: Weekly', + '---', + '', + 'Generated meeting notes.', + ].join('\n')); + + const withoutFlag = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + noExtract: true, + }); + expect(withoutFlag.status).toBe('up_to_date'); + expect(await engine.getPage('meetings/weekly')).toBeNull(); + + const withFlag = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + noExtract: true, + includeGitignored: true, + }); + expect(withFlag.added).toBe(1); + + const page = await engine.getPage('meetings/weekly'); + expect(page).not.toBeNull(); + expect(page!.title).toBe('Weekly'); + }); + test('detached HEAD skips git pull and ingests local working-tree files', async () => { const { performSync } = await import('../src/commands/sync.ts'); const seeded = await performSync(engine, {