fix(sync): include gitignored files on request (#3431)

Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com>

src/commands/import.ts

@@ -59,6 +59,11 @@ export async function runImport(
      * Threaded by performFullSync for `gbrain sync --exclude`.
      */
     exclude?: string[];
+    /**
+     * Opt out of the git-visible fast path and walk the filesystem directly,
+     * so markdown/code files matched by .gitignore can still be imported.
+     */
+    includeGitignored?: boolean;
     /**
      * #753/#774 monorepo subdir-source support: when set, slugs and
      * `source_path` are computed relative to this root (the git repo root)
@@ -71,6 +76,7 @@ export async function runImport(
   const noEmbed = args.includes('--no-embed');
   const fresh = args.includes('--fresh');
   const jsonOutput = args.includes('--json');
+  const includeGitignored = args.includes('--include-gitignored') || opts.includeGitignored === true;
 
   // T7 (D9): refuse cleanly when init persisted the deferred-setup sentinel,
   // unless the user is explicitly skipping embedding via `--no-embed` (in
@@ -185,7 +191,7 @@ export async function runImport(
   const dirArg = args.find((a, i) => !a.startsWith('--') && !flagValues.has(i));
 
   if (!dirArg) {
-    console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--json]');
+    console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--include-gitignored] [--json]');
     process.exit(1);
   }
   // #1728: capture the import target ONCE as an absolute real path. Every
@@ -209,7 +215,7 @@ export async function runImport(
   const strategy: SyncStrategy = opts.strategy ?? 'markdown';
   const _walkT0 = Date.now();
   console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
-  let allFiles = collectSyncableFiles(dir, { strategy });
+  let allFiles = collectSyncableFiles(dir, { strategy, includeGitignored });
   console.error(
     `[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
   );
@@ -545,6 +551,7 @@ function resolveMaxWalkDepth(): number {
 
 interface CollectOpts {
   strategy?: SyncStrategy;
+  includeGitignored?: boolean;
 }
 
 /**
@@ -675,8 +682,10 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
   // vendored data/fixtures). `--cached --others --exclude-standard` = tracked
   // PLUS untracked-not-ignored, so uncommitted source is still indexed. Non-git
   // dirs (or git unavailable) fall through to the FS walk below.
-  const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
-  if (gitFiles) return gitFiles;
+  if (!opts.includeGitignored) {
+    const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
+    if (gitFiles) return gitFiles;
+  }
 
   const maxDepth = resolveMaxWalkDepth();
   const visitedInodes = new Map<string, true>();

src/commands/sync.ts

@@ -239,11 +239,12 @@ export interface SyncResult {
 export function estimateSourceTreeTokens(
   localPath: string,
   strategy: 'markdown' | 'code' | 'auto',
+  opts: { includeGitignored?: boolean } = {},
 ): { tokens: number; files: number } {
   let tokens = 0;
   let files = 0;
   try {
-    const fileList = collectSyncableFiles(localPath, { strategy });
+    const fileList = collectSyncableFiles(localPath, { strategy, includeGitignored: opts.includeGitignored });
     for (const fullPath of fileList) {
       try {
         const stat = statSync(fullPath);
@@ -376,6 +377,7 @@ export function estimateInlineNewTokens(
     chunker_version: string | null;
   }>,
   currentChunkerVersion: string,
+  opts: { forceFullTree?: boolean } = {},
 ): InlineEstimate {
   let tokens = 0;
   let changedSources = 0;
@@ -398,6 +400,14 @@ export function estimateInlineNewTokens(
     const strategy = cfg.strategy ?? 'markdown';
     const localPath = src.local_path;
 
+    if (opts.forceFullTree) {
+      tokens += estimateSourceTreeTokens(localPath, strategy, { includeGitignored: true }).tokens;
+      changedSources++;
+      hadCeiling = true;
+      ceilingReasons.push('include_gitignored');
+      continue;
+    }
+
     // Rung 2: chunker drift forces a full re-chunk → full re-embed. CEILING.
     if (src.chunker_version !== currentChunkerVersion) {
       ceiling(localPath, strategy, 'chunker_drift');
@@ -542,6 +552,7 @@ interface CostGateContext {
   jsonOut: boolean;
   yesFlag: boolean;
   full: boolean;
+  includeGitignored?: boolean;
   /** Message prefix ('sync --all' | 'sync'). */
   label: string;
 }
@@ -626,7 +637,9 @@ async function runInlineCostGate(
   }
 
   // ── Inline path ───────────────────────────────────────────────
-  const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION));
+  const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION), {
+    forceFullTree: ctx.includeGitignored === true,
+  });
   // D7A: `--full` runs `performFullSync` → `runEmbedCore({stale:true})`, which
   // sweeps the pre-existing stale backlog INLINE on top of the delta. Price it.
   const costUsd = estimateEmbeddingCostUsd(inline.tokens) + (full ? staleCostUsd : 0);
@@ -764,6 +777,11 @@ export interface SyncOpts {
    * matching the #1433 metafile posture).
    */
   exclude?: string[];
+  /**
+   * Include files matched by .gitignore. Git cannot report untracked ignored
+   * changes in diffs, so sync uses the full filesystem walker when this is set.
+   */
+  includeGitignored?: boolean;
   /**
    * Number of parallel workers for the import phase. When > 1, each worker
    * gets its own small Postgres connection pool and files are dispatched via
@@ -2170,6 +2188,14 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
     return performFullSync(engine, fullSyncRoots, headCommit, opts);
   }
 
+  if (opts.includeGitignored) {
+    slog(
+      `[sync] --include-gitignored: running full filesystem reconcile because ` +
+      `git diff cannot report untracked ignored files.`,
+    );
+    return performFullSync(engine, fullSyncRoots, headCommit, opts);
+  }
+
   // v0.42.x (#1794): resumable incremental sync — resolve the PINNED target.
   // last_commit advances only at FULL import completion, so a killed run keeps
   // lastCommit fixed and the checkpoint key stable across every resume even as
@@ -3569,7 +3595,10 @@ async function performFullSync(
   // code --dry-run` always reported zero files even when ~1500 code
   // files were waiting.
   if (opts.dryRun) {
-    let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' });
+    let allFiles = collectSyncableFiles(syncScopeRoot, {
+      strategy: opts.strategy ?? 'markdown',
+      includeGitignored: opts.includeGitignored,
+    });
     if (opts.exclude && opts.exclude.length > 0) {
       allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
     }
@@ -3603,6 +3632,7 @@ async function performFullSync(
   const { runImport } = await import('./import.ts');
   const importArgs = [syncScopeRoot];
   if (opts.noEmbed) importArgs.push('--no-embed');
+  if (opts.includeGitignored) importArgs.push('--include-gitignored');
   if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency));
   // v0.31.2: thread strategy through so code-strategy first sync
   // actually enumerates code files (closes bug 1).
@@ -3616,6 +3646,7 @@ async function performFullSync(
     strategy: opts.strategy,
     sourceId: opts.sourceId,
     exclude: opts.exclude,
+    includeGitignored: opts.includeGitignored,
     slugRoot,
     // issue #1939: performFullSync owns the failure ledger + bookmark via the
     // shared gate below; don't let runImport double-record or write its own.
@@ -3728,7 +3759,10 @@ async function performFullSync(
     // #774: scoped syncs store git-root-relative source_paths (slugRoot), so
     // relativize the walk to the same base — otherwise every page mismatches
     // and the mass-delete valve trips on a perfectly healthy scoped source.
-    const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' })
+    const currentFiles = collectSyncableFiles(syncScopeRoot, {
+      strategy: opts.strategy ?? 'markdown',
+      includeGitignored: opts.includeGitignored,
+    })
       .map(abs => relative(slugRoot ?? syncScopeRoot, abs));
     const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>(
       `SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`,
@@ -4109,6 +4143,9 @@ Options:
                        subdirectory directly as --repo also works.
   --exclude <glob>     Exclude files matching the glob from sync (repeatable;
                        matched against the scope-relative path).
+  --include-gitignored Include otherwise-syncable files matched by .gitignore.
+                       Forces a full filesystem walk so periodic syncs see
+                       ignored untracked content.
   --dry-run            Show what would be synced without writing.
   --skip-failed        Acknowledge previously-recorded sync failures so
                        the bookmark can advance past unparseable files.
@@ -4159,6 +4196,7 @@ See also:
   const skipFailed = args.includes('--skip-failed');
   const retryFailed = args.includes('--retry-failed');
   const noSchemaPack = args.includes('--no-schema-pack'); // v0.41.37.0 #1569
+  const includeGitignored = args.includes('--include-gitignored');
   const syncAll = args.includes('--all');
   const jsonOut = args.includes('--json');
   const yesFlag = args.includes('--yes');
@@ -4415,7 +4453,7 @@ See also:
     if (!noEmbed) {
       const mode = willEmbedSynchronously({ v2Enabled, serialFlag, noEmbed });
       const gate = await runInlineCostGate(engine, {
-        sources, mode, dryRun, jsonOut, yesFlag, full, label: 'sync --all',
+        sources, mode, dryRun, jsonOut, yesFlag, full, includeGitignored, label: 'sync --all',
       });
       if (gate.action === 'stop') return;
       autoDeferEmbeds = gate.autoDeferEmbeds;
@@ -4513,6 +4551,7 @@ See also:
         noEmbed: effectiveNoEmbed,
         noExtract,
         skipFailed, retryFailed, noSchemaPack,
+        includeGitignored,
         sourceId: src.id,
         strategy: cfg.strategy,
         concurrency,
@@ -4737,7 +4776,7 @@ See also:
   const singleSourceInterrupt = new AbortController();
   const onSingleSourceSigint = () => { try { singleSourceInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
   const opts: SyncOpts = {
-    repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId,
+    repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, includeGitignored, sourceId,
     strategy: strategyArg, concurrency,
     srcSubpath,
     exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
@@ -4766,7 +4805,7 @@ See also:
         chunker_version: gateRows[0].chunker_version,
       }];
       const gate = await runInlineCostGate(engine, {
-        sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, label: 'sync',
+        sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, includeGitignored, label: 'sync',
       });
       if (gate.action === 'stop') return;
       if (gate.autoDeferEmbeds) {
@@ -4976,6 +5015,7 @@ export async function syncOneSource(
     noSchemaPack?: boolean;
     /** v0.42.7 #1696: propagate --no-extract into every per-source sync. */
     noExtract?: boolean;
+    includeGitignored?: boolean;
   },
 ): Promise<{ result: SyncResult; log: string }> {
   const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' };
@@ -4990,6 +5030,7 @@ export async function syncOneSource(
     skipFailed: shared.skipFailed,
     retryFailed: shared.retryFailed,
     noSchemaPack: shared.noSchemaPack,
+    includeGitignored: shared.includeGitignored,
     sourceId: src.id,
     strategy: cfg.strategy,
     concurrency: shared.concurrency,

test/import-git-fastpath-prune.test.ts

@@ -87,4 +87,27 @@ describe('#2607 — git fast path excludes what incremental sync excludes', () =
     }
     expect(files.length).toBeGreaterThan(0);
   });
+
+  test('--include-gitignored falls back to filesystem walk for ignored content', () => {
+    const ignoredRepo = mkdtempSync(join(tmpdir(), 'gbrain-include-ignored-'));
+    try {
+      execSync('git init', { cwd: ignoredRepo, stdio: 'pipe' });
+      writeFileSync(join(ignoredRepo, '.gitignore'), 'Meetings/\n');
+      writeFileSync(join(ignoredRepo, 'notes.md'), '---\ntitle: Notes\n---\nbody\n');
+      mkdirSync(join(ignoredRepo, 'Meetings'), { recursive: true });
+      writeFileSync(join(ignoredRepo, 'Meetings/weekly.md'), '---\ntitle: Weekly\n---\nbody\n');
+
+      const toRel = (files: string[]) => files.map((f) => relative(ignoredRepo, f));
+      const defaultFiles = toRel(collectSyncableFiles(ignoredRepo, { strategy: 'markdown' }));
+      const includeIgnored = toRel(collectSyncableFiles(ignoredRepo, {
+        strategy: 'markdown',
+        includeGitignored: true,
+      }));
+
+      expect(defaultFiles).not.toContain('Meetings/weekly.md');
+      expect(includeIgnored).toContain('Meetings/weekly.md');
+    } finally {
+      rmSync(ignoredRepo, { recursive: true, force: true });
+    }
+  });
 });

test/sync.test.ts

@@ -464,6 +464,59 @@ describe('performSync dry-run never writes', () => {
     expect(typeof result.embedded).toBe('number');
   });
 
+  test('--include-gitignored imports ignored files even when git HEAD is unchanged', async () => {
+    const { performSync } = await import('../src/commands/sync.ts');
+    const first = await performSync(engine, {
+      repoPath,
+      noPull: true,
+      noEmbed: true,
+      noExtract: true,
+    });
+    expect(first.status).toBe('first_sync');
+
+    writeFileSync(join(repoPath, '.gitignore'), 'Meetings/\n');
+    execSync('git add .gitignore && git commit -m "ignore generated meetings"', { cwd: repoPath, stdio: 'pipe' });
+    const checkpoint = await performSync(engine, {
+      repoPath,
+      noPull: true,
+      noEmbed: true,
+      noExtract: true,
+    });
+    expect(checkpoint.status).toBe('up_to_date');
+
+    mkdirSync(join(repoPath, 'Meetings'), { recursive: true });
+    writeFileSync(join(repoPath, 'Meetings/weekly.md'), [
+      '---',
+      'type: meeting',
+      'title: Weekly',
+      '---',
+      '',
+      'Generated meeting notes.',
+    ].join('\n'));
+
+    const withoutFlag = await performSync(engine, {
+      repoPath,
+      noPull: true,
+      noEmbed: true,
+      noExtract: true,
+    });
+    expect(withoutFlag.status).toBe('up_to_date');
+    expect(await engine.getPage('meetings/weekly')).toBeNull();
+
+    const withFlag = await performSync(engine, {
+      repoPath,
+      noPull: true,
+      noEmbed: true,
+      noExtract: true,
+      includeGitignored: true,
+    });
+    expect(withFlag.added).toBe(1);
+
+    const page = await engine.getPage('meetings/weekly');
+    expect(page).not.toBeNull();
+    expect(page!.title).toBe('Weekly');
+  });
+
   test('detached HEAD skips git pull and ingests local working-tree files', async () => {
     const { performSync } = await import('../src/commands/sync.ts');
     const seeded = await performSync(engine, {