Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 c3e9daa18f test(sync): pin the config-fingerprint gate wiring end-to-end (#2156)
The primitives (compute/read/write) were pinned but nothing exercised the
actual performSync gate: glob drift with git HEAD unchanged must break out
of the 'Already up to date' early return, force a full re-walk that imports
newly-included files, and settle back to up_to_date once re-stamped.
Deleting the configMismatch term from the gate now fails a committed test.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 11:10:48 -07:00
f28613178a feat(sources): persist + honor --include / --exclude globs across sync and lint (#2156)
Takeover of PR #2157, rebased onto current master.

- gbrain sources add --include/--exclude (repeatable) persist into
  sources.config.include_globs / exclude_globs; every subsequent sync
  of the source honors them.
- gbrain sync gains --include (allow-list counterpart to the existing
  #753/#774 --exclude), merged with the source row's persisted globs.
  Include/exclude are matched scope-relative in both the incremental
  and full-sync paths (import.ts gains the include filter); exclusion
  stays conservative (never deletes previously-imported pages).
- gbrain lint gains --include/--exclude and auto-lifts the persisted
  source globs when the lint target matches a source's local_path.
- Migration v125 (renumbered from the PR's v117/v120): a
  sources.config_fingerprint column caches a SHA-256 of the
  walk-affecting config fields (strategy + globs) so changing globs on
  an already-synced source forces a full re-walk instead of "Already
  up to date" (git HEAD unchanged). NULL = never stamped; first
  post-upgrade sync stamps quietly.

Deviations from #2157 while rebasing: globs are NOT threaded through
isSyncable/unsyncableReason in the incremental path — the unsyncable
cleanup loop deletes pages for non-metafile classifications, which
would violate the documented conservative #1433 posture; instead
include mirrors master's existing scope-relative excluded() helper.
CLI --exclude/--include now merge (union) with persisted config globs
rather than being replaced by them.

Fixes #2156
Takeover of #2157

Co-authored-by: brettdavies <brettdavies@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:36:26 -07:00
31 changed files with 1531 additions and 1907 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -7,7 +7,7 @@
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
<script type="module" crossorigin src="/admin/assets/index-BpDk4NI4.js"></script>
<script type="module" crossorigin src="/admin/assets/index-CoGEje3-.js"></script>
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
</head>
<body>
+2 -6
View File
@@ -5,14 +5,13 @@ import { AgentsPage } from './pages/Agents';
import { RequestLogPage } from './pages/RequestLog';
import { CalibrationPage } from './pages/Calibration';
import { JobsWatchPage } from './pages/JobsWatch';
import { SourcesPage } from './pages/Sources';
import { api } from './api';
type Page = 'login' | 'dashboard' | 'agents' | 'sources' | 'log' | 'calibration' | 'jobs';
type Page = 'login' | 'dashboard' | 'agents' | 'log' | 'calibration' | 'jobs';
function getPage(): Page {
const hash = window.location.hash.replace('#', '') || 'dashboard';
if (['login', 'dashboard', 'agents', 'sources', 'log', 'calibration', 'jobs'].includes(hash)) return hash as Page;
if (['login', 'dashboard', 'agents', 'log', 'calibration', 'jobs'].includes(hash)) return hash as Page;
return 'dashboard';
}
@@ -55,8 +54,6 @@ export function App() {
onClick={() => navigate('dashboard')}>Dashboard</a>
<a className={`nav-item ${page === 'agents' ? 'active' : ''}`}
onClick={() => navigate('agents')}>Agents</a>
<a className={`nav-item ${page === 'sources' ? 'active' : ''}`}
onClick={() => navigate('sources')}>Sources</a>
<a className={`nav-item ${page === 'log' ? 'active' : ''}`}
onClick={() => navigate('log')}>Request Log</a>
<a className={`nav-item ${page === 'calibration' ? 'active' : ''}`}
@@ -86,7 +83,6 @@ export function App() {
<main className="main">
{page === 'dashboard' && <DashboardPage />}
{page === 'agents' && <AgentsPage />}
{page === 'sources' && <SourcesPage />}
{page === 'log' && <RequestLogPage />}
{page === 'calibration' && <CalibrationPage />}
{page === 'jobs' && <JobsWatchPage />}
-18
View File
@@ -52,22 +52,4 @@ export const api = {
apiFetchText(`/admin/api/calibration/charts/${encodeURIComponent(type)}${holder ? `?holder=${encodeURIComponent(holder)}` : ''}`),
// v0.41 D2 — live minion-jobs dashboard snapshot.
jobsWatch: () => apiFetch('/admin/api/jobs/watch'),
// v0.41.29 Sources tab + federated-read management
sources: () => apiFetch('/admin/api/sources'),
agentsFederatedRead: () => apiFetch('/admin/api/agents/federated-read'),
grantRead: (clientId: string, sourceId: string) =>
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/grant-read`, {
method: 'POST',
body: JSON.stringify({ source_id: sourceId }),
}),
revokeRead: (clientId: string, sourceId: string) =>
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/revoke-read`, {
method: 'POST',
body: JSON.stringify({ source_id: sourceId }),
}),
setFederatedRead: (clientId: string, sourceIds: string[]) =>
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/set-federated-read`, {
method: 'POST',
body: JSON.stringify({ source_ids: sourceIds }),
}),
};
-196
View File
@@ -381,16 +381,8 @@ function CredentialsModal({ credentials, onClose }: {
);
}
interface FederationState {
source_id: string | null;
federated_read: string[];
}
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
const [federation, setFederation] = useState<FederationState | null>(null);
const [allSources, setAllSources] = useState<string[]>([]);
const [showFederation, setShowFederation] = useState(false);
const copy = (text: string) => navigator.clipboard.writeText(text);
const serverUrl = window.location.origin;
@@ -398,30 +390,6 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
const isOAuth = agent.auth_type === 'oauth';
const agentName = agent.name || agent.client_name || 'unknown';
// Lazy-load federation state when the drawer opens for an OAuth client.
// The /admin/api/agents endpoint doesn't carry source_id / federated_read,
// so we fetch /admin/api/agents/federated-read separately and pair by id.
useEffect(() => {
if (!isOAuth || !cid) return;
let cancelled = false;
Promise.all([
api.agentsFederatedRead().catch(() => ({ clients: [] })),
api.sources().catch(() => ({ sources: [] })),
]).then(([feds, srcs]: any) => {
if (cancelled) return;
const me = (feds.clients || []).find((c: any) => c.client_id === cid);
setFederation(me ? { source_id: me.source_id, federated_read: me.federated_read || [] } : null);
setAllSources((srcs.sources || []).map((s: any) => s.source_id));
});
return () => { cancelled = true; };
}, [cid, isOAuth]);
const reloadFederation = async () => {
const feds: any = await api.agentsFederatedRead().catch(() => ({ clients: [] }));
const me = (feds.clients || []).find((c: any) => c.client_id === cid);
setFederation(me ? { source_id: me.source_id, federated_read: me.federated_read || [] } : null);
};
// For API keys, we can't show the actual token (it was shown once at creation).
// For OAuth, we show the client_id and tell them to use their secret.
@@ -585,34 +553,6 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
</div>
{isOAuth && federation && (
<>
<div className="section-title" style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
<span>Federation</span>
<button
className="btn btn-secondary"
style={{ padding: '4px 10px', fontSize: 12 }}
onClick={() => setShowFederation(true)}
>
Manage reads
</button>
</div>
<div style={{ display: 'grid', gridTemplateColumns: '120px 1fr', gap: '6px 12px', fontSize: 13 }}>
<span style={{ color: 'var(--text-secondary)' }}>Write source</span>
<span className="mono">{federation.source_id || '(none)'}</span>
<span style={{ color: 'var(--text-secondary)' }}>Federated reads</span>
<span style={{ fontSize: 12 }}>
{federation.federated_read.length === 0
? <span style={{ color: 'var(--text-muted)' }}>(empty no federated reads)</span>
: federation.federated_read.map((s) => (
<span key={s} className="badge badge-read" style={{ marginRight: 4, marginBottom: 2 }}>{s}</span>
))
}
</span>
</div>
</>
)}
{/*
Config Export visible for both auth_type=oauth AND auth_type=api_key.
Claude Code + Cursor + JSON tabs render real snippets regardless
@@ -688,142 +628,6 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
)}
</div>
</div>
{showFederation && federation && (
<FederationModal
clientId={cid}
clientName={agentName}
allSources={allSources}
currentReads={federation.federated_read}
writeSource={federation.source_id}
onClose={() => setShowFederation(false)}
onSaved={async () => {
await reloadFederation();
setShowFederation(false);
}}
/>
)}
</>
);
}
/**
* FederationModal admin counterpart of `gbrain auth set-federated-read`.
* Source checkbox list; "Save" submits the full new list via the
* race-safe atomic SQL path in setFederatedReadCore. Per the CLI's
* documented contract, this is wholesale-replace semantics concurrent
* grant/revoke from a CLI operator would be last-writer-wins against
* a Save here.
*/
function FederationModal({
clientId, clientName, allSources, currentReads, writeSource, onClose, onSaved,
}: {
clientId: string;
clientName: string;
allSources: string[];
currentReads: string[];
writeSource: string | null;
onClose: () => void;
onSaved: () => Promise<void> | void;
}) {
const [selected, setSelected] = useState<Set<string>>(new Set(currentReads));
const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const toggle = (id: string) => {
const next = new Set(selected);
if (next.has(id)) next.delete(id); else next.add(id);
setSelected(next);
};
const handleSave = async () => {
setSaving(true);
setError(null);
try {
await api.setFederatedRead(clientId, Array.from(selected));
await onSaved();
} catch (e: any) {
setError(e.message || 'save failed');
setSaving(false);
}
};
// Union: all known sources + any current reads not in the source list
// (e.g. orphan entries from before the source was deleted). The latter
// surface as "(missing source)" so operators can revoke them.
const allKnown = new Set([...allSources, ...currentReads]);
const ordered = Array.from(allKnown).sort();
return (
<div className="modal-overlay" onClick={onClose}>
<div className="modal" onClick={(e) => e.stopPropagation()} style={{ maxWidth: 520 }}>
<div className="modal-header">
<div style={{ fontSize: 16, fontWeight: 600 }}>Manage federated reads</div>
<div style={{ fontSize: 13, color: 'var(--text-secondary)', marginTop: 4 }}>
<strong>{clientName}</strong> pick which sources this client can read in addition to its
{writeSource ? <> write source <code className="mono">{writeSource}</code></> : <> write source</>}.
</div>
</div>
<div className="modal-body" style={{ maxHeight: '50vh', overflowY: 'auto' }}>
{ordered.length === 0 && (
<div style={{ color: 'var(--text-muted)', fontSize: 13 }}>
No sources registered. Use <code>gbrain sources add &lt;id&gt; --path &lt;dir&gt;</code> from the CLI first.
</div>
)}
{ordered.map((id) => {
const isOrphan = !allSources.includes(id);
const isWriteSource = id === writeSource;
return (
<label
key={id}
style={{
display: 'flex',
alignItems: 'center',
gap: 10,
padding: '8px 10px',
borderBottom: '1px solid var(--border)',
cursor: 'pointer',
fontSize: 13,
}}
>
<input
type="checkbox"
checked={selected.has(id)}
onChange={() => toggle(id)}
style={{ width: 16, height: 16, margin: 0, flexShrink: 0, cursor: 'pointer' }}
/>
<span className="mono" style={{ flex: 1, minWidth: 0, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{id}</span>
{isWriteSource && <span className="badge badge-write" style={{ fontSize: 10, flexShrink: 0 }}>write source</span>}
{isOrphan && <span className="badge badge-danger" style={{ fontSize: 10, flexShrink: 0 }}>missing source</span>}
</label>
);
})}
</div>
{error && (
<div style={{
background: 'rgba(239,68,68,0.08)',
border: '1px solid rgba(239,68,68,0.3)',
color: '#ef4444',
padding: '10px 12px',
borderRadius: 6,
margin: '12px 0',
fontSize: 12,
}}>
{error}
</div>
)}
<div className="modal-footer">
<button type="button" className="btn btn-secondary" onClick={onClose} disabled={saving}>Cancel</button>
<button
type="button"
className="btn btn-primary"
onClick={handleSave}
disabled={saving}
>
{saving ? 'Saving…' : `Save (${selected.size} source${selected.size === 1 ? '' : 's'})`}
</button>
</div>
</div>
</div>
);
}
-195
View File
@@ -1,195 +0,0 @@
import React, { useState, useEffect } from 'react';
import { api } from '../api';
interface SourceRow {
source_id: string;
name: string;
local_path: string | null;
sync_enabled: boolean;
last_sync_at: string | null;
staleness_hours: number | null;
staleness_class: 'fresh' | 'stale' | 'severe' | 'unknown';
last_commit: string | null;
pages: number;
chunks_total: number;
chunks_unembedded: number;
embedding_coverage_pct: number;
}
interface FederatedClient {
client_id: string;
client_name: string;
source_id: string | null;
federated_read: string[];
}
function timeAgo(iso: string | null): string {
if (!iso) return 'never';
const s = Math.floor((Date.now() - new Date(iso).getTime()) / 1000);
if (s < 0) return 'in the future?';
if (s < 60) return 'just now';
if (s < 3600) return `${Math.floor(s / 60)}m ago`;
if (s < 86400) return `${Math.floor(s / 3600)}h ago`;
return `${Math.floor(s / 86400)}d ago`;
}
function stalenessColor(cls: string): string {
switch (cls) {
case 'fresh': return '#4ade80';
case 'stale': return '#fbbf24';
case 'severe': return '#ef4444';
default: return 'var(--text-muted)';
}
}
function coverageColor(pct: number): string {
if (pct >= 99) return '#4ade80';
if (pct >= 90) return '#fbbf24';
return '#ef4444';
}
export function SourcesPage() {
const [sources, setSources] = useState<SourceRow[]>([]);
const [clients, setClients] = useState<FederatedClient[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const load = async () => {
setLoading(true);
setError(null);
try {
const [srcReport, clientsResp] = await Promise.all([
api.sources(),
api.agentsFederatedRead(),
]);
setSources(srcReport.sources || []);
setClients(clientsResp.clients || []);
} catch (e: any) {
setError(e.message || 'load failed');
} finally {
setLoading(false);
}
};
useEffect(() => { load(); }, []);
// Reverse-lookup: for each source, which clients can read it?
const readersBySource = (sourceId: string): string[] =>
clients.filter((c) => c.federated_read.includes(sourceId)).map((c) => c.client_name);
// Reverse-lookup: which clients WRITE to this source (source_id == sourceId)?
const writersBySource = (sourceId: string): string[] =>
clients.filter((c) => c.source_id === sourceId).map((c) => c.client_name);
return (
<div style={{ padding: 24, maxWidth: 1200 }}>
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', marginBottom: 24 }}>
<h1 style={{ fontSize: 24, margin: 0 }}>Sources</h1>
<button
onClick={load}
style={{
background: 'transparent',
border: '1px solid var(--border)',
color: 'var(--text-secondary)',
padding: '6px 12px',
borderRadius: 6,
fontSize: 12,
cursor: 'pointer',
}}
>
Refresh
</button>
</div>
{loading && <div style={{ color: 'var(--text-muted)' }}>Loading</div>}
{error && (
<div style={{
background: 'rgba(239,68,68,0.08)',
border: '1px solid rgba(239,68,68,0.3)',
color: '#ef4444',
padding: 12,
borderRadius: 6,
marginBottom: 16,
fontSize: 13,
}}>
Failed to load sources: {error}
</div>
)}
{!loading && !error && sources.length === 0 && (
<div style={{ color: 'var(--text-muted)', padding: 16 }}>
No active sources with a local_path. Use{' '}
<code style={{ background: 'var(--bg-elevated)', padding: '2px 6px', borderRadius: 4 }}>
gbrain sources add &lt;id&gt; --path &lt;dir&gt;
</code>{' '}
to register one.
</div>
)}
{!loading && !error && sources.length > 0 && (
<div style={{ overflowX: 'auto' }}>
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
<thead>
<tr style={{ borderBottom: '1px solid var(--border)', textAlign: 'left', color: 'var(--text-muted)' }}>
<th style={{ padding: '10px 12px' }}>ID</th>
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Pages</th>
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Chunks</th>
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Embed%</th>
<th style={{ padding: '10px 12px' }}>Last Sync</th>
<th style={{ padding: '10px 12px' }}>Writers</th>
<th style={{ padding: '10px 12px' }}>Readers (federated)</th>
</tr>
</thead>
<tbody>
{sources.map((s) => {
const readers = readersBySource(s.source_id);
const writers = writersBySource(s.source_id);
return (
<tr key={s.source_id} style={{ borderBottom: '1px solid var(--border)' }}>
<td style={{ padding: '10px 12px', fontFamily: 'JetBrains Mono, monospace' }}>
<div>{s.source_id}</div>
{s.name !== s.source_id && (
<div style={{ fontSize: 11, color: 'var(--text-muted)', fontFamily: 'inherit' }}>{s.name}</div>
)}
</td>
<td style={{ padding: '10px 12px', textAlign: 'right', fontFamily: 'JetBrains Mono, monospace' }}>{s.pages.toLocaleString()}</td>
<td style={{ padding: '10px 12px', textAlign: 'right', fontFamily: 'JetBrains Mono, monospace' }}>{s.chunks_total.toLocaleString()}</td>
<td style={{ padding: '10px 12px', textAlign: 'right', color: coverageColor(s.embedding_coverage_pct), fontFamily: 'JetBrains Mono, monospace' }}>
{s.embedding_coverage_pct.toFixed(0)}%
</td>
<td style={{ padding: '10px 12px', color: s.local_path == null ? 'var(--text-muted)' : stalenessColor(s.staleness_class) }}>
{s.local_path == null ? 'push-only' : timeAgo(s.last_sync_at)}
</td>
<td style={{ padding: '10px 12px', fontSize: 12, color: 'var(--text-secondary)' }}>
{writers.length === 0 ? <span style={{ color: 'var(--text-muted)' }}>none</span> : writers.join(', ')}
</td>
<td style={{ padding: '10px 12px', fontSize: 12, color: 'var(--text-secondary)' }}>
{readers.length === 0 ? <span style={{ color: 'var(--text-muted)' }}>none</span> : readers.join(', ')}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
)}
<div style={{
marginTop: 24,
padding: 12,
background: 'var(--bg-elevated)',
border: '1px solid var(--border)',
borderRadius: 6,
fontSize: 12,
color: 'var(--text-muted)',
lineHeight: 1.6,
}}>
<strong style={{ color: 'var(--text-secondary)' }}>Two scopes per OAuth client:</strong>{' '}
<em>Writers</em> = clients with this source as their <code>source_id</code> (write authority).{' '}
<em>Readers</em> = clients with this source in their <code>federated_read</code> list (read access via federation).
Manage federation per-client from the <a href="#agents" style={{ color: '#60a5fa' }}>Agents</a> tab using the
"Manage reads" action.
</div>
</div>
);
}
+38
View File
@@ -115,6 +115,7 @@ Full subcommand reference:
```
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated] [--force]
[--include <glob>...] [--exclude <glob>...]
Register a source. id: [a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?
--path must be a git repo (or a subdirectory of one) — see
"The git requirement for --path sources" below. --force
@@ -131,6 +132,43 @@ gbrain sources federate <id>
gbrain sources unfederate <id>
```
## Filtering what gets synced (--include / --exclude)
`--include` and `--exclude` on `gbrain sources add` accept repeatable glob
patterns and are honored by every subsequent sync AND lint of the source.
Common Obsidian vault setups need to exclude authoring scaffolding so it
doesn't pollute search:
```bash
# Skip Templates/, Drafts/, and the smart-env sidecar; everything else syncs.
gbrain sources add vault \
--path ~/Documents/vault --federated \
--exclude 'Templates/**' \
--exclude 'Drafts/**' \
--exclude '.smart-env/**'
# Or: only sync the people/ and companies/ subtrees of a CRM vault.
gbrain sources add crm \
--path ~/Documents/crm --no-federated \
--include 'people/**' \
--include 'companies/**'
```
Both persist into `sources.config.include_globs` / `exclude_globs` arrays.
The filter runs `include` first, then `exclude`, so a path inside
`people/**` is still rejected if it also matches `exclude_globs`. Globs use
the same matcher as the rest of gbrain's sync classifier (`matchesAnyGlob`
in `src/core/sync.ts`) and are matched against the source-root-relative
path. Exclusion is conservative: it never deletes previously-imported pages.
`gbrain sync --include <glob> --exclude <glob>` and
`gbrain lint <dir> --include <glob> --exclude <glob>` take the same
repeatable flags for one-off scope changes; for lint the persisted source
globs are auto-applied when the lint target matches a source's `local_path`.
Changing the persisted globs on an existing source triggers a full re-walk
on the next sync (the source's config fingerprint invalidates the
"already up to date" gate).
## The git requirement for --path sources
Every `--path` source must be a git repository (or live inside one — a
+3 -3
View File
@@ -1,13 +1,13 @@
// AUTO-GENERATED — do not edit by hand.
// Run `bun run scripts/build-admin-embedded.ts` to regenerate.
// Source: admin/dist/ at 2026-07-22.
// Source: admin/dist/ at 2026-05-27.
//
// Bun resolves the file: imports to a path that works at runtime even
// inside a compiled binary (`bun build --compile`). The manifest maps
// the request path the express handler sees to (resolved-path, mime).
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
import A_0_assets_index_BpDk4NI4_js from '../admin/dist/assets/index-BpDk4NI4.js' with { type: 'file' };
import A_0_assets_index_CoGEje3__js from '../admin/dist/assets/index-CoGEje3-.js' with { type: 'file' };
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
import A_1_assets_index_GxkWX7v3_css from '../admin/dist/assets/index-GxkWX7v3.css' with { type: 'file' };
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
@@ -19,7 +19,7 @@ export interface AdminAsset {
}
export const ADMIN_ASSETS: Record<string, AdminAsset> = {
"/admin/assets/index-BpDk4NI4.js": { path: A_0_assets_index_BpDk4NI4_js as unknown as string, mime: "application/javascript; charset=utf-8" },
"/admin/assets/index-CoGEje3-.js": { path: A_0_assets_index_CoGEje3__js as unknown as string, mime: "application/javascript; charset=utf-8" },
"/admin/assets/index-GxkWX7v3.css": { path: A_1_assets_index_GxkWX7v3_css as unknown as string, mime: "text/css; charset=utf-8" },
"/admin/index.html": { path: A_2_index_html as unknown as string, mime: "text/html; charset=utf-8" },
};
+1 -586
View File
@@ -24,8 +24,6 @@ import { loadConfig, toEngineConfig } from '../core/config.ts';
import { createEngine } from '../core/engine-factory.ts';
import type { BrainEngine } from '../core/engine.ts';
import { sqlQueryForEngine, executeRawJsonb, type SqlQuery } from '../core/sql-query.ts';
import { pgArray } from '../core/oauth-provider.ts';
import { assertValidSourceId } from '../core/source-id.ts';
function hashToken(token: string): string {
return createHash('sha256').update(token).digest('hex');
@@ -167,100 +165,6 @@ async function list() {
});
}
/**
* `gbrain auth list-clients [--json]` read surface for OAuth 2.1 clients.
*
* The existing `gbrain auth list` shows LEGACY bearer tokens from
* `access_tokens`; this is the parallel for v0.26+ OAuth clients. Separate
* commands rather than merged output because the two models have different
* field sets (legacy: lifecycle dates; OAuth: scopes + source_id +
* federated_read).
*
* Human output is card-style (multi-line per client) instead of a fixed-
* width table federated_read can hold many ids per client and a wide
* single-line layout truncates / wraps badly on terminals < 200 cols.
* JSON output uses a `schema_version: 1` envelope; additive only.
*/
async function listClients(args: string[]) {
const json = args.includes('--json');
const includeDeleted = args.includes('--include-deleted');
await withConfiguredSql(async (sql) => {
// Codex finding #2 (medium): default-hide soft-deleted clients so admin
// soft-deletes are honored by the CLI surface. Opt-in via flag.
const rows = includeDeleted
? await sql`
SELECT client_id, client_name, scope, source_id, federated_read,
grant_types, created_at, deleted_at
FROM oauth_clients
ORDER BY client_name
`
: await sql`
SELECT client_id, client_name, scope, source_id, federated_read,
grant_types, created_at, deleted_at
FROM oauth_clients
WHERE deleted_at IS NULL
ORDER BY client_name
`;
if (json) {
const clients = rows.map((r) => ({
client_id: String(r.client_id),
client_name: String(r.client_name),
scope: r.scope == null ? null : String(r.scope),
source_id: r.source_id == null ? null : String(r.source_id),
federated_read: Array.isArray(r.federated_read)
? (r.federated_read as string[]).map(String)
: [],
grant_types: Array.isArray(r.grant_types)
? (r.grant_types as string[]).map(String)
: [],
created_at:
r.created_at instanceof Date
? r.created_at.toISOString()
: r.created_at == null
? null
: String(r.created_at),
deleted_at:
r.deleted_at instanceof Date
? r.deleted_at.toISOString()
: r.deleted_at == null
? null
: String(r.deleted_at),
}));
process.stdout.write(JSON.stringify({ schema_version: 1, clients }, null, 2) + '\n');
return;
}
if (rows.length === 0) {
console.log(
includeDeleted
? 'No OAuth clients found (including deleted). Register one: gbrain auth register-client <name>'
: 'No active OAuth clients found. Register one: gbrain auth register-client <name>'
+ '\n(Use --include-deleted to also show soft-deleted clients.)',
);
return;
}
for (let i = 0; i < rows.length; i++) {
const r = rows[i];
const fed = Array.isArray(r.federated_read)
? (r.federated_read as string[]).map(String)
: [];
const grants = Array.isArray(r.grant_types)
? (r.grant_types as string[]).map(String)
: [];
const deletedAt = r.deleted_at;
const status = deletedAt == null
? ''
: ` [SOFT-DELETED ${deletedAt instanceof Date ? deletedAt.toISOString() : String(deletedAt)}]`;
console.log(`${sanitizeForTerminal(String(r.client_name))}${status}`);
console.log(` client_id: ${sanitizeForTerminal(String(r.client_id))}`);
console.log(` scope: ${r.scope == null ? '(none)' : sanitizeForTerminal(String(r.scope))}`);
console.log(` grant types: ${grants.length ? sanitizeForTerminal(grants.join(', ')) : '(none)'}`);
console.log(` write source: ${r.source_id == null ? '(none)' : sanitizeForTerminal(String(r.source_id))}`);
console.log(` federated: ${fed.length ? sanitizeForTerminal(fed.join(', ')) : '(empty)'}`);
if (i < rows.length - 1) console.log('');
}
});
}
async function revoke(name: string) {
if (!name) { console.error('Usage: auth revoke <name>'); process.exit(1); }
await withConfiguredSql(async (sql) => {
@@ -397,475 +301,6 @@ async function test(url: string, token: string) {
console.log(`\n🧠 Your brain is live! (${elapsed}s)`);
}
/**
* Strip ANSI escapes + C0/C1 control characters from a string before
* printing it to the operator's terminal. Defense for the
* codex-flagged terminal-control-injection class: a client_name or
* source_id registered via DCR with `\x1b[2J` (clear-screen) or
* `\x1b]0;TITLE\x07` (OSC title-change) would poison
* `gbrain auth list-clients` output otherwise.
*
* Replaces unsafe bytes with their `\xNN` hex escape so the operator
* sees that something weird is in the field, instead of silent
* mutilation. Tab and newline are preserved as-is so legitimate
* multi-line values render.
*/
export function sanitizeForTerminal(s: string): string {
// ALL C0/C1 controls + DEL get escaped. Codex re-review caught that
// preserving `\n` lets a DCR-registered client_name spoof additional
// human-output lines in list-clients (a real attack — newline in the
// name visually adds a fake row to the operator's terminal). Tab is
// also escaped for the same reason — field-separator spoofing.
// C0: 0x00-0x1F. DEL: 0x7F. C1: 0x80-0x9F.
return s.replace(/[\x00-\x1f\x7f-\x9f]/g, (ch) =>
`\\x${ch.charCodeAt(0).toString(16).padStart(2, '0')}`,
);
}
export interface ResolvedClient {
client_id: string;
client_name: string;
source_id: string | null;
federated_read: string[];
deleted_at: Date | string | null;
}
export type FederatedReadOutcome =
| { kind: 'noop'; reason: 'already-granted' | 'not-present' | 'same-list'; client: ResolvedClient; current: string[] }
| { kind: 'updated'; client: ResolvedClient; before: string[]; after: string[] };
/**
* Resolve an OAuth client by client_id (exact) or client_name (unique).
* Errors on no-match and on ambiguous client_name (>1 row). client_id
* takes precedence if a long hash is passed and matches, returns
* immediately without ever querying by name.
*
* Legacy bearer tokens in `access_tokens` are NOT searched. Federated read
* scope is an OAuth-client concept (oauth_clients.federated_read column);
* legacy bearers have no source scope.
*/
/**
* Resolve an OAuth client. Codex finding #2 (medium): default-hide
* soft-deleted clients so admin-soft-deleted rows aren't mutated by the
* CLI. The `includeDeleted` opt is reserved for future read-side surfaces;
* grant/revoke/set ALWAYS filter active rows only.
*/
export async function resolveClient(
sql: SqlQuery,
nameOrId: string,
opts: { includeDeleted?: boolean } = {},
): Promise<ResolvedClient> {
const allowDeleted = opts.includeDeleted === true;
const byId = allowDeleted
? await sql`
SELECT client_id, client_name, source_id, federated_read, deleted_at
FROM oauth_clients WHERE client_id = ${nameOrId} LIMIT 1
`
: await sql`
SELECT client_id, client_name, source_id, federated_read, deleted_at
FROM oauth_clients WHERE client_id = ${nameOrId} AND deleted_at IS NULL LIMIT 1
`;
if (byId.length === 1) return normalizeClientRow(byId[0]);
const byName = allowDeleted
? await sql`
SELECT client_id, client_name, source_id, federated_read, deleted_at
FROM oauth_clients WHERE client_name = ${nameOrId}
`
: await sql`
SELECT client_id, client_name, source_id, federated_read, deleted_at
FROM oauth_clients WHERE client_name = ${nameOrId} AND deleted_at IS NULL
`;
if (byName.length === 0) {
throw new Error(
`No active OAuth client found with name or id "${nameOrId}". ` +
`Run \`gbrain auth register-client <name>\` to create one, ` +
`or \`gbrain auth list-clients\` to see what exists. ` +
`(Soft-deleted clients are hidden by default.)`,
);
}
if (byName.length > 1) {
const ids = byName.map((r) => ` ${String(r.client_id)}`).join('\n');
throw new Error(
`Multiple active OAuth clients named "${nameOrId}". Pass the full client_id instead:\n${ids}`,
);
}
return normalizeClientRow(byName[0]);
}
function normalizeClientRow(row: Record<string, unknown>): ResolvedClient {
const fed = row.federated_read;
return {
client_id: String(row.client_id),
client_name: String(row.client_name),
source_id: row.source_id == null ? null : String(row.source_id),
federated_read: Array.isArray(fed) ? (fed as string[]).map(String) : [],
deleted_at: row.deleted_at == null
? null
: (row.deleted_at as Date | string),
};
}
/**
* Validate the source_id shape AND DB existence. Codex finding #3 (medium):
* a manually-INSERTed source row with weird chars (e.g. comma, quote)
* would otherwise land in oauth_clients.federated_read as a never-deletable
* malformed entry. Fail at the boundary before the existence query so
* malformed input gets the validator's hint, not a "does not exist" hint
* pointing at a non-creatable id.
*/
export async function assertSourceExists(sql: SqlQuery, sourceId: string): Promise<void> {
assertValidSourceId(sourceId);
const rows = await sql`SELECT id FROM sources WHERE id = ${sourceId} LIMIT 1`;
if (rows.length === 0) {
throw new Error(
`Source "${sourceId}" does not exist. Run \`gbrain sources list\` to see registered sources, ` +
`or \`gbrain sources add ${sourceId}\` to create it.`,
);
}
}
/**
* Atomic append: array_append + NOT-ANY guard so the row-lock fully
* serializes concurrent grant/revoke against the same client. Codex
* finding #1 (HIGH): the previous read-modify-write shape allowed a
* concurrent revoke to be silently UNDONE by a racing grant.
*
* Returns the post-write federated_read array, or null when no rows
* matched (already-granted, soft-deleted, or missing client). Callers
* disambiguate via prior resolveClient + includes() check.
*
* `WHERE deleted_at IS NULL` is part of the atomic guard so a client
* soft-deleted between resolveClient and the UPDATE can't be mutated.
*/
async function appendFederatedReadAtomic(
sql: SqlQuery,
clientId: string,
sourceId: string,
): Promise<string[] | null> {
const rows = await sql`
UPDATE oauth_clients
SET federated_read = array_append(federated_read, ${sourceId})
WHERE client_id = ${clientId}
AND deleted_at IS NULL
AND NOT (${sourceId} = ANY(federated_read))
RETURNING federated_read
`;
if (rows.length === 0) return null;
const fed = rows[0].federated_read;
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
}
/**
* Atomic remove: array_remove + ANY guard. Same race-correctness story
* as appendFederatedReadAtomic. Returns post-write array or null.
*/
async function removeFederatedReadAtomic(
sql: SqlQuery,
clientId: string,
sourceId: string,
): Promise<string[] | null> {
const rows = await sql`
UPDATE oauth_clients
SET federated_read = array_remove(federated_read, ${sourceId})
WHERE client_id = ${clientId}
AND deleted_at IS NULL
AND ${sourceId} = ANY(federated_read)
RETURNING federated_read
`;
if (rows.length === 0) return null;
const fed = rows[0].federated_read;
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
}
/**
* Wholesale array overwrite for `set-federated-read`. Honors the
* deleted_at filter. Last-writer-wins semantics under concurrent
* `set` calls is acceptable the user is asserting "this exact list"
* intent; concurrent set+set just means whichever ran second wins.
* Concurrent set+grant or set+revoke is also last-writer-wins, which
* is the documented contract for `set`.
*/
async function replaceFederatedReadAtomic(
sql: SqlQuery,
clientId: string,
next: string[],
): Promise<string[] | null> {
// TEXT[] binding via pgArray() string-literal escaping (see helper
// for the security note). Our narrow SqlQuery surface
// (src/core/sql-query.ts) doesn't bind JS arrays directly.
const literal = pgArray(next);
const rows = await sql`
UPDATE oauth_clients
SET federated_read = ${literal}
WHERE client_id = ${clientId}
AND deleted_at IS NULL
RETURNING federated_read
`;
if (rows.length === 0) return null;
const fed = rows[0].federated_read;
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
}
/**
* Pure helper: dedupe a comma-separated source-id list while preserving
* insertion order. Empty input empty array. Exported so the CLI parser
* and tests share one normalizer.
*/
export function parseSourceCsv(csv: string): string[] {
const requested = csv.split(',').map((s) => s.trim()).filter(Boolean);
const seen = new Set<string>();
const out: string[] = [];
for (const s of requested) {
if (!seen.has(s)) {
seen.add(s);
out.push(s);
}
}
return out;
}
export interface FederatedReadOpts {
/** When true, compute the outcome but skip the persisting UPDATE. */
dryRun?: boolean;
}
/**
* Core: append a source to the client's federated_read.
*
* Atomicity contract (Codex finding #1, HIGH):
* The actual write goes through `appendFederatedReadAtomic` which
* serializes at the row-lock so concurrent grant/revoke against the
* same client cannot lose updates. The race vector that previously
* silently restored revoked access is closed: under two operators
* racing `revoke-read sensitive` + `grant-read harmless`, postgres
* serializes the two UPDATEs and BOTH ops apply (sensitive removed,
* harmless added), instead of one clobbering the other.
*
* The reported `before` is the snapshot at resolveClient time, which
* may be stale relative to a concurrent racer. The `after` reflects
* the post-UPDATE state from RETURNING (always fresh).
*/
export async function grantReadCore(
sql: SqlQuery,
nameOrId: string,
sourceId: string,
opts: FederatedReadOpts = {},
): Promise<FederatedReadOutcome> {
const client = await resolveClient(sql, nameOrId);
await assertSourceExists(sql, sourceId);
if (client.federated_read.includes(sourceId)) {
return { kind: 'noop', reason: 'already-granted', client, current: client.federated_read };
}
if (opts.dryRun) {
// Compute the would-be result without touching the row. Last-known
// snapshot is best-effort under concurrent writes.
const projected = [...client.federated_read, sourceId];
return { kind: 'updated', client, before: client.federated_read, after: projected };
}
const after = await appendFederatedReadAtomic(sql, client.client_id, sourceId);
if (after === null) {
// Two equivalent failure modes: (a) racing grant-read already added
// the source and the NOT-ANY guard suppressed our UPDATE, or
// (b) the client was soft-deleted between resolveClient and UPDATE.
// (a) is the more common path. Re-resolve to confirm + report.
const reresolved = await resolveClient(sql, client.client_id, { includeDeleted: true });
if (reresolved.deleted_at != null) {
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
}
return { kind: 'noop', reason: 'already-granted', client: reresolved, current: reresolved.federated_read };
}
return { kind: 'updated', client, before: client.federated_read, after };
}
/**
* Core: remove a source from the client's federated_read. Atomic via
* array_remove + ANY-guard. Same race-correctness rationale as
* grantReadCore concurrent ops serialize at the row lock.
*/
export async function revokeReadCore(
sql: SqlQuery,
nameOrId: string,
sourceId: string,
opts: FederatedReadOpts = {},
): Promise<FederatedReadOutcome> {
const client = await resolveClient(sql, nameOrId);
if (!client.federated_read.includes(sourceId)) {
return { kind: 'noop', reason: 'not-present', client, current: client.federated_read };
}
if (opts.dryRun) {
const projected = client.federated_read.filter((s) => s !== sourceId);
return { kind: 'updated', client, before: client.federated_read, after: projected };
}
const after = await removeFederatedReadAtomic(sql, client.client_id, sourceId);
if (after === null) {
// Same disambiguation as grant: either a concurrent revoke already
// removed the source (most common) or the client was soft-deleted.
const reresolved = await resolveClient(sql, client.client_id, { includeDeleted: true });
if (reresolved.deleted_at != null) {
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
}
return { kind: 'noop', reason: 'not-present', client: reresolved, current: reresolved.federated_read };
}
return { kind: 'updated', client, before: client.federated_read, after };
}
/**
* Core: replace the whole federated_read list. Idempotent on same list.
*
* Race semantics: wholesale-overwrite + deleted_at guard. Concurrent
* set+set is last-writer-wins (documented contract for `set` the
* operator is asserting the exact list). Concurrent set+grant or
* set+revoke is also last-writer-wins. If a strict-merge semantics is
* needed, use grant-read / revoke-read individually.
*/
export async function setFederatedReadCore(
sql: SqlQuery,
nameOrId: string,
sourceCsv: string,
opts: FederatedReadOpts = {},
): Promise<FederatedReadOutcome> {
const next = parseSourceCsv(sourceCsv);
const client = await resolveClient(sql, nameOrId);
for (const s of next) {
await assertSourceExists(sql, s);
}
const prev = client.federated_read;
const same = prev.length === next.length && prev.every((v, i) => v === next[i]);
if (same) {
return { kind: 'noop', reason: 'same-list', client, current: prev };
}
if (opts.dryRun) {
return { kind: 'updated', client, before: prev, after: next };
}
const after = await replaceFederatedReadAtomic(sql, client.client_id, next);
if (after === null) {
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
}
return { kind: 'updated', client, before: prev, after };
}
function printOutcome(
verb: 'grant' | 'revoke' | 'set',
sourceArg: string,
outcome: FederatedReadOutcome,
dryRun: boolean,
): void {
// Terminal-injection defense (Codex finding #5, low): a client_name
// registered via DCR with ANSI escapes or control chars would
// otherwise poison this output. Sanitize ALL strings that round-trip
// from the DB before printing.
const s = sanitizeForTerminal;
const prefix = dryRun ? '[dry-run] ' : '';
if (outcome.kind === 'noop') {
const name = s(outcome.client.client_name);
if (outcome.reason === 'already-granted') {
console.log(`${prefix}No change: "${name}" already reads "${s(sourceArg)}".`);
} else if (outcome.reason === 'not-present') {
console.log(`${prefix}No change: "${name}" did not read "${s(sourceArg)}".`);
} else {
console.log(`${prefix}No change: "${name}" federated_read already matches.`);
}
console.log(` federated_read: ${outcome.current.map(s).join(', ') || '(empty)'}`);
return;
}
const { client, before, after } = outcome;
const name = s(client.client_name);
const wouldOrDid = dryRun ? 'Would' : 'Did';
if (verb === 'grant') {
console.log(`${prefix}${wouldOrDid} grant: "${name}" can now read "${s(sourceArg)}".`);
console.log(` federated_read: ${after.map(s).join(', ')}`);
} else if (verb === 'revoke') {
console.log(`${prefix}${wouldOrDid} revoke: "${name}" no longer reads "${s(sourceArg)}".`);
console.log(` federated_read: ${after.map(s).join(', ') || '(empty — client has no federated reads)'}`);
} else {
console.log(`${prefix}${wouldOrDid} update "${name}" federated_read:`);
console.log(` before: ${before.map(s).join(', ') || '(empty)'}`);
console.log(` after: ${after.map(s).join(', ') || '(empty)'}`);
}
if (after.length === 0) {
console.log(
'Warning: client now reads no sources via federation. Queries through this ' +
'client will only see content scoped explicitly via its write source.',
);
}
}
/**
* Strip `--dry-run` from a positional-arg list. Returns the filtered list
* plus the flag value. Kept positional-tolerant the existing
* `auth grant-read alice source` shape MUST keep working, AND
* `auth grant-read alice source --dry-run` AND `auth grant-read --dry-run alice source`.
*/
export function extractDryRun(args: string[]): { dryRun: boolean; rest: string[] } {
let dryRun = false;
const rest: string[] = [];
for (const a of args) {
if (a === '--dry-run') {
dryRun = true;
continue;
}
rest.push(a);
}
return { dryRun, rest };
}
async function grantRead(args: string[]): Promise<void> {
const { dryRun, rest } = extractDryRun(args);
const [nameOrId, sourceId] = rest;
if (!nameOrId || !sourceId) {
console.error('Usage: gbrain auth grant-read <client-name-or-id> <source-id> [--dry-run]');
process.exit(1);
}
try {
await withConfiguredSql(async (sql) => {
const outcome = await grantReadCore(sql, nameOrId, sourceId, { dryRun });
printOutcome('grant', sourceId, outcome, dryRun);
});
} catch (e: any) {
console.error('Error:', e.message);
process.exit(1);
}
}
async function revokeRead(args: string[]): Promise<void> {
const { dryRun, rest } = extractDryRun(args);
const [nameOrId, sourceId] = rest;
if (!nameOrId || !sourceId) {
console.error('Usage: gbrain auth revoke-read <client-name-or-id> <source-id> [--dry-run]');
process.exit(1);
}
try {
await withConfiguredSql(async (sql) => {
const outcome = await revokeReadCore(sql, nameOrId, sourceId, { dryRun });
printOutcome('revoke', sourceId, outcome, dryRun);
});
} catch (e: any) {
console.error('Error:', e.message);
process.exit(1);
}
}
async function setFederatedRead(args: string[]): Promise<void> {
const { dryRun, rest } = extractDryRun(args);
const [nameOrId, sourceCsv] = rest;
if (!nameOrId || sourceCsv === undefined) {
console.error(
'Usage: gbrain auth set-federated-read <client-name-or-id> <source-id1,source-id2,...> [--dry-run]',
);
console.error('Pass an empty string ("") to clear all federated reads.');
process.exit(1);
}
try {
await withConfiguredSql(async (sql) => {
const outcome = await setFederatedReadCore(sql, nameOrId, sourceCsv, { dryRun });
printOutcome('set', sourceCsv, outcome, dryRun);
});
} catch (e: any) {
console.error('Error:', e.message);
process.exit(1);
}
}
async function revokeClient(clientId: string) {
if (!clientId) {
console.error('Usage: auth revoke-client <client_id>');
@@ -884,7 +319,7 @@ async function revokeClient(clientId: string) {
console.error(`No client found with id "${clientId}"`);
process.exit(1);
}
console.log(`OAuth client revoked: "${sanitizeForTerminal(String(rows[0].client_name))}" (${clientId})`);
console.log(`OAuth client revoked: "${rows[0].client_name}" (${clientId})`);
console.log('Tokens and authorization codes purged via cascade.');
});
} catch (e: any) {
@@ -1005,15 +440,6 @@ export function parseRegisterClientArgs(args: string[]): RegisterClientArgs {
if (!grantTypesSet && out.redirectUris.length > 0) {
out.grantTypes = ['authorization_code', 'refresh_token'];
}
// Codex re-review (medium): validate source_id shape at the CLI boundary
// so register-client can't seed malformed entries into source_id /
// federated_read that subsequent grant/revoke/set commands can't manage.
assertValidSourceId(out.sourceId);
if (out.federatedRead) {
for (const s of out.federatedRead) {
assertValidSourceId(s);
}
}
return out;
}
@@ -1131,10 +557,6 @@ export async function runAuth(args: string[]): Promise<void> {
}
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
case 'revoke-client': await revokeClient(rest[0]); return;
case 'list-clients': await listClients(rest); return;
case 'grant-read': await grantRead(rest); return;
case 'revoke-read': await revokeRead(rest); return;
case 'set-federated-read': await setFederatedRead(rest); return;
case 'test': {
const tokenIdx = rest.indexOf('--token');
const url = rest.find(a => !a.startsWith('--') && a !== rest[tokenIdx + 1]);
@@ -1172,13 +594,6 @@ Usage:
--bound-max-concurrent <n> Bound submit_agent concurrency (default: 1)
--budget-usd-per-day <usd> Bound submit_agent daily spend cap
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
gbrain auth list-clients [--json] List OAuth 2.1 clients with scope + write source + federated_read.
gbrain auth grant-read <name|client_id> <source-id> [--dry-run]
Add a source to the client's federated_read list (idempotent).
gbrain auth revoke-read <name|client_id> <source-id> [--dry-run]
Remove a source from the client's federated_read list (idempotent).
gbrain auth set-federated-read <name|client_id> "<id1,id2,...>" [--dry-run]
Replace the client's whole federated_read list. Pass "" to clear.
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
`);
}
+11
View File
@@ -53,6 +53,13 @@ export async function runImport(
strategy?: SyncStrategy;
sourceId?: string;
managedBookmark?: boolean;
/**
* #2156: allow-list glob patterns only dir-relative paths matching at
* least one pattern are imported. Applied BEFORE `exclude`. Threaded by
* performFullSync from `gbrain sync --include` / the source row's
* persisted `config.include_globs`.
*/
include?: string[];
/**
* #753/#774: glob patterns to exclude from the import (same semantics as
* `isSyncable`'s `exclude` matched against the dir-relative path).
@@ -215,6 +222,10 @@ export async function runImport(
);
const fileTypeLabel = strategy === 'code' ? 'code'
: strategy === 'auto' ? 'syncable' : 'markdown';
// #2156: apply --include allow-list globs first (threaded by performFullSync).
if (opts.include && opts.include.length > 0) {
allFiles = allFiles.filter(abs => matchesAnyGlob(relative(dir, abs), opts.include));
}
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
if (opts.exclude && opts.exclude.length > 0) {
const beforeExclude = allFiles.length;
+172 -12
View File
@@ -17,7 +17,7 @@
*/
import { readFileSync, writeFileSync, readdirSync, statSync, lstatSync, existsSync } from 'fs';
import { join, relative } from 'path';
import { join, relative, resolve } from 'path';
import { isAborted } from '../core/abort-check.ts';
import { parseMarkdown, type ParseValidationCode } from '../core/markdown.ts';
import {
@@ -26,7 +26,9 @@ import {
DEFAULT_BYTES_WARN,
} from '../core/content-sanity.ts';
import { loadOperatorLiterals } from '../core/content-sanity-literals.ts';
import { loadConfig, loadConfigWithEngine, gbrainPath } from '../core/config.ts';
import { loadConfig, loadConfigWithEngine, toEngineConfig, gbrainPath } from '../core/config.ts';
import { matchesAnyGlob } from '../core/sync.ts';
import { parseGlobList } from './sync.ts';
import type { BrainEngine } from '../core/engine.ts';
export interface LintIssue {
@@ -378,21 +380,89 @@ async function resolveLintContentSanity(
};
}
/** Collect markdown files from a directory */
function collectPages(dir: string): string[] {
/** Collect markdown files from a directory.
*
* When `opts.include` or `opts.exclude` are set, each candidate `.md` path's
* POSIX-style relative path (relative to `dir`) is matched against the same
* glob semantics sync uses (`matchesAnyGlob`). `include` allow-lists;
* `exclude` deny-lists. Empty or undefined arrays leave the filter
* unengaged. Symmetric with `isSyncable` in `src/core/sync.ts` so a
* source-config `exclude_globs` honored by `gbrain sync` is also honored
* by `gbrain lint` against the same dir.
*/
function collectPages(
dir: string,
opts: { include?: string[]; exclude?: string[] } = {},
): string[] {
const { include, exclude } = opts;
const haveInclude = !!(include && include.length > 0);
const haveExclude = !!(exclude && exclude.length > 0);
const pages: string[] = [];
function walk(d: string) {
for (const entry of readdirSync(d)) {
if (entry.startsWith('.') || entry.startsWith('_')) continue;
const full = join(d, entry);
if (lstatSync(full).isDirectory()) walk(full);
else if (entry.endsWith('.md')) pages.push(full);
else if (entry.endsWith('.md')) {
if (haveInclude || haveExclude) {
// Match against the path RELATIVE to `dir` (the source root),
// normalized to POSIX separators by matchesAnyGlob. A
// source-config glob like `Resources/veriff/**` is anchored at
// the source root; matching against the absolute path would
// require the user to anchor on their `$HOME` or repo prefix,
// which is brittle.
const rel = relative(dir, full);
if (haveInclude && !matchesAnyGlob(rel, include)) continue;
if (haveExclude && matchesAnyGlob(rel, exclude)) continue;
}
pages.push(full);
}
}
}
walk(dir);
return pages.sort();
}
/** Look up the source row whose `local_path` resolves to the same absolute
* directory as `target`, and return its persisted `include_globs` /
* `exclude_globs` as parsed string arrays. Returns an empty object when no
* matching source exists, when the row has no globs configured, or when the
* lookup throws (best-effort auto-resolution must never break standalone
* lint on brains without a sources table).
*
* Mirrors how `syncOneSource` lifts the same fields off `src.config` before
* threading them into `SyncOpts.include` / `SyncOpts.exclude`.
*/
async function resolveSourceGlobsForTarget(
engine: BrainEngine,
target: string,
): Promise<{ include?: string[]; exclude?: string[] }> {
try {
const absTarget = resolve(target);
const rows = await engine.executeRaw<{ config: unknown }>(
`SELECT config FROM sources
WHERE archived IS NOT TRUE
AND local_path IS NOT NULL
AND local_path = $1
LIMIT 1`,
[absTarget],
);
if (rows.length === 0) return {};
const cfg = (rows[0].config && typeof rows[0].config === 'object')
? rows[0].config as Record<string, unknown>
: {};
return {
include: parseGlobList(cfg.include_globs),
exclude: parseGlobList(cfg.exclude_globs),
};
} catch {
// Engine not connected, sources table missing on a fresh brain, RLS
// denial in an unusual scope — all best-effort. Lint proceeds without
// filtering rather than fail-closed.
return {};
}
}
export interface LintOpts {
target: string;
fix?: boolean;
@@ -414,6 +484,22 @@ export interface LintOpts {
* yields + checks this every 200 pages.
*/
signal?: AbortSignal;
/**
* Glob filters threaded into the file walker. When set, paths relative to
* `target` are matched against the patterns using the same semantics as
* `gbrain sync` (`matchesAnyGlob` in `src/core/sync.ts`). `include`
* allow-lists; `exclude` deny-lists; both unset == no filter.
*
* When BOTH are unset AND `engine` is provided, `runLintCore` attempts to
* auto-resolve them from the `sources` row whose `local_path` matches
* `target` symmetric with `syncOneSource`, so a user who has run
* `gbrain sources add --exclude 'Resources/veriff/**'` sees the same
* exclusion applied to `gbrain lint <same-dir>` and to the cycle.lint
* phase without restating it on every invocation. Explicit caller-supplied
* arrays always win over the source-row lift.
*/
include?: string[];
exclude?: string[];
}
export interface LintResult {
@@ -440,7 +526,21 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
}
const isSingleFile = statSync(opts.target).isFile();
const pages = isSingleFile ? [opts.target] : collectPages(opts.target);
// Resolve glob filters. Explicit caller-supplied include/exclude win;
// otherwise lift from `sources.config.{include,exclude}_globs` when an
// engine is available and the target matches a known source's local_path.
// Single-file lints skip the resolve entirely — globs are a directory
// walk concern.
let include = opts.include;
let exclude = opts.exclude;
const haveExplicit = (include && include.length > 0) || (exclude && exclude.length > 0);
if (!isSingleFile && !haveExplicit && opts.engine) {
const resolved = await resolveSourceGlobsForTarget(opts.engine, opts.target);
include = resolved.include;
exclude = resolved.exclude;
}
const pages = isSingleFile ? [opts.target] : collectPages(opts.target, { include, exclude });
// Resolve content-sanity config once for this lint run (D1: lift DB
// config when reachable). Caller can pre-pass via opts.contentSanity
@@ -491,14 +591,27 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
}
export async function runLint(args: string[]) {
const target = args.find(a => !a.startsWith('--'));
const target = args.find(a => !a.startsWith('--') && !args[args.indexOf(a) - 1]?.match(/^--(include|exclude)$/));
const doFix = args.includes('--fix');
const dryRun = args.includes('--dry-run');
// Parse repeatable `--include <glob>` and `--exclude <glob>` flags.
// Symmetric with `gbrain sources add --include / --exclude` from PR #2157;
// explicit flags here override the source-config lift performed below for
// dir-mode lints.
const cliInclude: string[] = [];
const cliExclude: string[] = [];
for (let i = 0; i < args.length; i++) {
if (args[i] === '--include' && i + 1 < args.length) cliInclude.push(args[++i]);
else if (args[i] === '--exclude' && i + 1 < args.length) cliExclude.push(args[++i]);
}
if (!target) {
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run]');
console.error(' --fix Auto-fix fixable issues (LLM preambles, code fences)');
console.error(' --dry-run Preview fixes without writing');
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run] [--include <glob>]... [--exclude <glob>]...');
console.error(' --fix Auto-fix fixable issues (LLM preambles, code fences)');
console.error(' --dry-run Preview fixes without writing');
console.error(' --include <glob> Repeatable; only lint paths matching at least one pattern');
console.error(' --exclude <glob> Repeatable; skip paths matching any pattern (applied after --include)');
process.exit(1);
}
@@ -510,7 +623,44 @@ export async function runLint(args: string[]) {
// Single file or directory — print human detail as we go, then rely on
// Core for the aggregate numbers at the end.
const isSingleFile = statSync(target).isFile();
const pages = isSingleFile ? [target] : collectPages(target);
// Resolve glob filters for directory lints. Explicit CLI flags win;
// otherwise lift from `sources.config.{include,exclude}_globs` matching
// `target`. Connect a transient engine for the lookup only when (a) no
// explicit flags were passed AND (b) file/env config suggests an engine is
// available — mirrors the connect-disconnect pattern in
// `resolveLintContentSanity` (issue #1678: standalone CLI never shares the
// db.ts singleton, so create + dispose here is safe).
let runInclude: string[] | undefined = cliInclude.length > 0 ? cliInclude : undefined;
let runExclude: string[] | undefined = cliExclude.length > 0 ? cliExclude : undefined;
if (!isSingleFile && runInclude === undefined && runExclude === undefined) {
const base = loadConfig();
if (base?.database_url || base?.database_path) {
try {
const { createEngine } = await import('../core/engine-factory.ts');
const { connectWithRetry } = await import('../core/db.ts');
const engineCfg = toEngineConfig(base);
const engine = await createEngine(engineCfg);
try {
// Use the same connect path the rest of the CLI uses
// (`connectEngine` in cli.ts). `engine.connect({})` with empty
// opts drops the URL — confirmed by direct probe. `noRetry: true`
// keeps the standalone lint snappy (no retry tax when the brain
// happens to be unreachable; auto-resolve degrades to no-filter).
await connectWithRetry(engine, engineCfg, { noRetry: true });
const lifted = await resolveSourceGlobsForTarget(engine, target);
runInclude = lifted.include;
runExclude = lifted.exclude;
} finally {
await engine.disconnect().catch(() => { /* best-effort */ });
}
} catch {
// best-effort; fall through to no-filter
}
}
}
const pages = isSingleFile ? [target] : collectPages(target, { include: runInclude, exclude: runExclude });
// Progress on stderr. Stdout keeps the per-issue human output it always had.
const { createProgress } = await import('../core/progress.ts');
@@ -557,7 +707,17 @@ export async function runLint(args: string[]) {
// produces canonical numbers for the summary line).
// Pass contentSanity through so runLintCore skips its own resolve
// (we already resolved once for the human-detail loop above).
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity });
// Pass include/exclude so the aggregate scope matches the human-detail
// walk above — otherwise the summary line reports the unfiltered count
// even though the per-page details were already filtered.
const result = await runLintCore({
target,
fix: doFix,
dryRun,
contentSanity,
include: runInclude,
exclude: runExclude,
});
console.log(`\n${result.pages_scanned} pages scanned. ${result.total_issues} issue(s) in ${result.pages_with_issues} page(s).`);
if (doFix) {
console.log(`${dryRun ? '(dry run) ' : ''}${result.total_fixed} auto-fixed.`);
-141
View File
@@ -365,42 +365,6 @@ export interface AgentClientSpend {
inflight_count: number;
}
/**
* `/admin/api/sources` source list the input rows for buildSyncStatusReport.
*
* Queries the JSONB config column directly (listSources doesn't carry it,
* but buildSyncStatusReport needs syncEnabled / strategy fields).
*
* Deliberately does NOT filter on local_path: in a push-only deployment
* (content arrives via MCP put_page / capture / ingest, not `gbrain sync`
* of a server checkout) every source has a null local_path filtering on
* it would empty both the Sources tab AND the federation source-picker.
* buildSyncStatusReport does no disk I/O, so null-local_path sources
* report fine (pages/chunks from SQL, staleness 'unknown' / never-synced).
*/
export async function queryAdminSources(engine: BrainEngine): Promise<
Array<{ id: string; name: string; local_path: string | null; config: Record<string, unknown> }>
> {
const rows = await engine.executeRaw<{
id: string;
name: string;
local_path: string | null;
config: Record<string, unknown> | string | null;
}>(
`SELECT id, name, local_path, config FROM sources
WHERE archived IS NOT TRUE
ORDER BY id`,
);
return rows.map((r) => ({
id: r.id,
name: r.name,
local_path: r.local_path,
config: typeof r.config === 'string'
? (JSON.parse(r.config) as Record<string, unknown>)
: (r.config ?? {}),
}));
}
export async function queryAgentClientSpend(engine: BrainEngine): Promise<AgentClientSpend[]> {
const sql = sqlQueryForEngine(engine);
const rows = await sql`
@@ -1547,111 +1511,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
}
});
// ---------------------------------------------------------------------------
// Sources tab — read-only view of registered sources with sync + embed
// coverage stats. Drives the admin SPA's `Sources` page.
//
// Returns the same shape `gbrain sources status --json` prints, so the
// SPA stays in lockstep with the CLI surface.
// ---------------------------------------------------------------------------
app.get('/admin/api/sources', requireAdmin, async (_req: Request, res: Response) => {
try {
const { buildSyncStatusReport } = await import('./sync.ts');
const report = await buildSyncStatusReport(engine, await queryAdminSources(engine));
res.json(report);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
res.status(503).json({ error: 'service_unavailable', detail: msg });
}
});
// ---------------------------------------------------------------------------
// Federated-read management (admin-side counterparts of the CLI commands
// `gbrain auth grant-read / revoke-read / set-federated-read`). All three
// route through the same *Core helpers as the CLI so race-safety,
// soft-delete filter, and source-id shape validation apply uniformly.
//
// The admin SPA's `Agents` page renders "Manage reads" actions per
// client backed by these endpoints.
// ---------------------------------------------------------------------------
app.get('/admin/api/agents/federated-read', requireAdmin, async (_req: Request, res: Response) => {
try {
const rows = await sql`
SELECT client_id, client_name, source_id, federated_read
FROM oauth_clients
WHERE deleted_at IS NULL
ORDER BY client_name
`;
const clients = rows.map((r) => ({
client_id: String(r.client_id),
client_name: String(r.client_name),
source_id: r.source_id == null ? null : String(r.source_id),
federated_read: Array.isArray(r.federated_read)
? (r.federated_read as string[]).map(String)
: [],
}));
res.json({ clients });
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
res.status(503).json({ error: 'service_unavailable', detail: msg });
}
});
app.post('/admin/api/agents/:clientId/grant-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
const clientId = String(req.params.clientId ?? '');
const sourceId = String(req.body?.source_id ?? '').trim();
if (!clientId || !sourceId) {
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_id body required' });
return;
}
try {
const { grantReadCore } = await import('./auth.ts');
const outcome = await grantReadCore(sql, clientId, sourceId);
res.json({ outcome });
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
res.status(400).json({ error: 'mutation_failed', detail: msg });
}
});
app.post('/admin/api/agents/:clientId/revoke-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
const clientId = String(req.params.clientId ?? '');
const sourceId = String(req.body?.source_id ?? '').trim();
if (!clientId || !sourceId) {
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_id body required' });
return;
}
try {
const { revokeReadCore } = await import('./auth.ts');
const outcome = await revokeReadCore(sql, clientId, sourceId);
res.json({ outcome });
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
res.status(400).json({ error: 'mutation_failed', detail: msg });
}
});
app.post('/admin/api/agents/:clientId/set-federated-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
const clientId = String(req.params.clientId ?? '');
const rawIds = req.body?.source_ids;
if (!clientId || !Array.isArray(rawIds)) {
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_ids[] body required' });
return;
}
// Encode the array as CSV so the same setFederatedReadCore signature
// (string CSV input) the CLI uses applies here. Empty array → empty
// string → clears the list.
const csv = rawIds.map((s) => String(s).trim()).filter(Boolean).join(',');
try {
const { setFederatedReadCore } = await import('./auth.ts');
const outcome = await setFederatedReadCore(sql, clientId, csv);
res.json({ outcome });
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
res.status(400).json({ error: 'mutation_failed', detail: msg });
}
});
// ---------------------------------------------------------------------------
// SSE live activity feed
// ---------------------------------------------------------------------------
+34 -1
View File
@@ -122,7 +122,8 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
if (!id) {
console.error(
'Usage: gbrain sources add <id> [--path <path> | --url <https-url>] ' +
'[--name <display>] [--federated|--no-federated] [--clone-dir <path>] [--force]',
'[--name <display>] [--federated|--no-federated] [--clone-dir <path>] [--force] ' +
'[--include <glob>...] [--exclude <glob>...]',
);
process.exit(2);
}
@@ -135,6 +136,12 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
let patFile: string | undefined;
let noHarden = false;
let force = false;
// Repeatable. `--include 'people/**' --include 'companies/**'` accumulates.
// Persisted into sources.config.include_globs / .exclude_globs and read at
// sync time by commands/sync.ts so `Templates/`, `.smart-env/`, `Drafts/`
// and other vault scaffolding can be skipped without renaming directories.
const includeGlobs: string[] = [];
const excludeGlobs: string[] = [];
for (let i = 1; i < args.length; i++) {
const a = args[i];
@@ -147,6 +154,24 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
if (a === '--pat-file') { patFile = args[++i]; continue; }
if (a === '--no-harden') { noHarden = true; continue; }
if (a === '--force') { force = true; continue; }
if (a === '--include') {
const v = args[++i];
if (!v || v.startsWith('--')) {
console.error('Error: --include requires a glob argument (e.g. --include "people/**")');
process.exit(2);
}
includeGlobs.push(v);
continue;
}
if (a === '--exclude') {
const v = args[++i];
if (!v || v.startsWith('--')) {
console.error('Error: --exclude requires a glob argument (e.g. --exclude "Templates/**")');
process.exit(2);
}
excludeGlobs.push(v);
continue;
}
console.error(`Unknown flag: ${a}`);
process.exit(2);
}
@@ -167,6 +192,8 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
federated,
cloneDir,
force,
includeGlobs: includeGlobs.length > 0 ? includeGlobs : undefined,
excludeGlobs: excludeGlobs.length > 0 ? excludeGlobs : undefined,
});
// Topology A discovery: if the just-added source carries a brain-resident
@@ -190,6 +217,12 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
console.log(
` federated: ${fed}${fed ? ' — appears in cross-source default search' : ' — only searched when explicitly named via --source'}`,
);
if (includeGlobs.length > 0) {
console.log(` include globs: ${includeGlobs.join(', ')}`);
}
if (excludeGlobs.length > 0) {
console.log(` exclude globs: ${excludeGlobs.join(', ')}`);
}
// v0.42.44 — auto-harden managed clones for git durability the moment a brain
// repo is added with a PAT. Best-effort: NEVER fail `add` if hardening fails.
+256 -18
View File
@@ -1,6 +1,7 @@
import { existsSync, readFileSync, writeFileSync, statSync, realpathSync } from 'fs';
import { execFileSync } from 'child_process';
import { join, relative } from 'path';
import { createHash } from 'crypto';
import type { BrainEngine } from '../core/engine.ts';
import { DELETE_BATCH_SIZE } from '../core/engine-constants.ts';
import { importFile } from '../core/import-file.ts';
@@ -756,12 +757,23 @@ export interface SyncOpts {
* are rejected before any git op runs.
*/
srcSubpath?: string;
/**
* #2156 glob patterns files must match to be synced (allow-list).
* Populated from the source row's persisted `config.include_globs`
* (set via `gbrain sources add --include <glob>`) or the repeatable
* `--include` CLI flag. Matched against the scope-relative path, same
* anchoring as `exclude`. `exclude` is applied after `include`: a path
* matching an include pattern is still rejected if it also matches an
* exclude pattern. Empty arrays are the same as undefined (no filter).
*/
include?: string[];
/**
* #753/#774 glob patterns for files to exclude from sync (repeatable
* `--exclude` on the CLI). Matched against the scope-relative path in both
* the full-sync and incremental paths. Excluded files are never imported;
* exclusion does NOT delete previously-imported pages (conservative,
* matching the #1433 metafile posture).
* `--exclude` on the CLI; #2156: also populated from the source row's
* persisted `config.exclude_globs`). Matched against the scope-relative
* path in both the full-sync and incremental paths. Excluded files are
* never imported; exclusion does NOT delete previously-imported pages
* (conservative, matching the #1433 metafile posture).
*/
exclude?: string[];
/**
@@ -1153,6 +1165,29 @@ function unique<T>(items: T[]): T[] {
// `src/core/sync-delta.ts` (re-imported below) so the inline cost estimator
// prices detached sources through the same code the executor imports them with.
/**
* Defensive parse for the JSONB-loaded `config.include_globs` / `config.exclude_globs`
* arrays read off the sources row. The column is a free-form JSONB and could
* contain anything coerce to a string-only array, drop empties, and return
* undefined when the result has no useful entries so the caller can decide
* not to engage glob-filtering at all.
*/
export function parseGlobList(value: unknown): string[] | undefined {
if (!Array.isArray(value)) return undefined;
const globs = value.filter((v): v is string => typeof v === 'string' && v.length > 0);
return globs.length > 0 ? globs : undefined;
}
/**
* Union of CLI-supplied glob patterns (one-off, this invocation) and the
* source row's persisted config globs (every sync). Deduped; undefined when
* neither side has entries so `SyncOpts` stays unset and no filter engages.
*/
export function mergeGlobs(cli: string[], persisted: string[] | undefined): string[] | undefined {
const merged = [...new Set([...cli, ...(persisted ?? [])])];
return merged.length > 0 ? merged : undefined;
}
// v0.18.0 Step 5: source-scoped sync state helpers. When opts.sourceId
// is set, read/write the per-source row instead of the global config
// keys. These wrappers centralize the branch so every read/write site
@@ -1315,6 +1350,125 @@ async function writeChunkerVersion(
);
}
/**
* #2157 follow-on: detect when sources.config has shifted in a way that
* affects which paths the walker will include this run. The "Already up
* to date" gate at performSync's git-HEAD equality check honored chunker
* version match but ignored config drift a user who changes
* `sources.config.exclude_globs` mid-life got "Already up to date" on
* the next sync because git HEAD was unchanged, with no observable
* effect until `gbrain sync --full`.
*
* Fingerprint covers exactly the walk-affecting fields that flow from
* `sources.config` into `SyncOpts` at the syncOneSource call site:
* `strategy`, `include_globs`, `exclude_globs`. CLI-supplied --include
* / --exclude overrides do NOT participate they are one-off scope
* changes, not source state, and shouldn't invalidate the row's
* checkpoint. (A user running `gbrain sync --exclude X` on a row whose
* stored config has no X is intentionally narrowing this one pass; on
* the next no-flags sync, the row config governs again.)
*
* Array order is normalized (alphabetical, post-defensive-parse) so
* `["a/**", "b/**"]` and `["b/**", "a/**"]` fingerprint identically.
* `parseGlobList` shares the same defensive coercion as the call site
* that builds SyncOpts, so hand-edited or pre-normalization rows
* fingerprint to the same shape the walker actually sees.
*/
export function computeSourceConfigFingerprint(rawConfig: unknown): string {
const cfg = (rawConfig || {}) as {
strategy?: unknown;
include_globs?: unknown;
exclude_globs?: unknown;
};
const canonical = JSON.stringify({
strategy: typeof cfg.strategy === 'string' ? cfg.strategy : null,
include_globs: (parseGlobList(cfg.include_globs) ?? []).slice().sort(),
exclude_globs: (parseGlobList(cfg.exclude_globs) ?? []).slice().sort(),
});
return createHash('sha256').update(canonical).digest('hex');
}
/**
* Read the per-source fingerprint stamp. NULL on pre-migration rows or
* sources that have never been synced the gate treats NULL as
* "fingerprint unknown" and skips the invalidation check so first-time
* post-upgrade syncs don't spuriously force-full.
*/
export async function readConfigFingerprint(
engine: BrainEngine,
sourceId: string | undefined,
): Promise<string | null> {
if (!sourceId) return null;
const rows = await engine.executeRaw<{ config_fingerprint: string | null }>(
`SELECT config_fingerprint FROM sources WHERE id = $1`,
[sourceId],
);
return rows[0]?.config_fingerprint ?? null;
}
export async function writeConfigFingerprint(
engine: BrainEngine,
sourceId: string | undefined,
fingerprint: string,
): Promise<void> {
if (!sourceId) return;
await engine.executeRaw(
`UPDATE sources SET config_fingerprint = $1 WHERE id = $2`,
[fingerprint, sourceId],
);
}
/**
* Read the raw `sources.config` value for the named source. Returns an
* empty object for missing or never-configured rows. The reader is
* defensive about legacy double-encoded JSONB rows (`{"federated":true}`
* stored as a JSON string scalar, the #2339 class) the engine's
* `r.config` may arrive as either a string or an object, and both are
* normalized to an object before the fingerprint computation walks the
* keys.
*/
async function readSourceConfig(
engine: BrainEngine,
sourceId: string | undefined,
): Promise<unknown> {
if (!sourceId) return {};
const rows = await engine.executeRaw<{ config: unknown }>(
`SELECT config FROM sources WHERE id = $1`,
[sourceId],
);
const raw = rows[0]?.config;
if (raw === null || raw === undefined) return {};
if (typeof raw === 'string') {
try { return JSON.parse(raw); } catch { return {}; }
}
return raw;
}
/**
* Read-hash-stamp wrapper for sync-completion sites outside the gate's
* scope (e.g. `performFullSync`'s `advanceFull` closure, which doesn't
* see `performSync`'s cached `currentConfigFp` because it's a separate
* function). Reads the row's current config and stamps a fresh
* fingerprint.
*
* Race note: if `sources.config` was mutated between the gate's read
* and this stamp, the freshly-read value wins. The walker still used
* the gate-time effective globs (already captured into `opts.include` /
* `opts.exclude` upstream), so the stamp can drift from what was
* actually walked. In practice mid-sync mutations are rare and the
* NEXT sync will re-evaluate against the latest config anyway, so the
* minor staleness is acceptable and avoids threading the gate-time
* fingerprint through every helper signature.
*/
async function stampSourceConfigFingerprint(
engine: BrainEngine,
sourceId: string | undefined,
): Promise<void> {
if (!sourceId) return;
const cfg = await readSourceConfig(engine, sourceId);
await writeConfigFingerprint(engine, sourceId, computeSourceConfigFingerprint(cfg));
}
/**
* v0.40 Federated Sync v2: `gbrain sync trigger --source <id> [--priority high|normal|low]`
*
@@ -2163,7 +2317,25 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
detachedWorkingTreeManifest.deleted.length > 0 ||
detachedWorkingTreeManifest.renamed.length > 0);
if (lastCommit === headCommit && !versionMismatch && !versionNeverSet && !hasDetachedWorkingTreeChanges) {
// #2157 follow-on: parallel gate for sources.config drift. Without
// this, changing `sources.config.exclude_globs` (or include_globs /
// strategy) on a synced source had no observable effect on the next
// sync because git HEAD was unchanged — the "Already up to date"
// branch below returned without re-walking. Mismatch path mirrors the
// chunker_version gate exactly so both kinds of drift route through
// the same `performFullSync` recovery.
//
// NULL stored fingerprint is "never stamped" (pre-v125 brain OR fresh
// source whose first sync hasn't completed yet). Treated as
// pass-through in the up-to-date check — first post-upgrade sync
// stamps the column quietly so subsequent passes have a baseline.
const storedConfigFp = await readConfigFingerprint(engine, opts.sourceId);
const currentSourceConfig = await readSourceConfig(engine, opts.sourceId);
const currentConfigFp = computeSourceConfigFingerprint(currentSourceConfig);
const configMismatch = storedConfigFp !== null && storedConfigFp !== currentConfigFp;
const configNeverStamped = storedConfigFp === null && opts.sourceId !== undefined;
if (lastCommit === headCommit && !versionMismatch && !versionNeverSet && !hasDetachedWorkingTreeChanges && !configMismatch) {
// v0.42.52.0 (PR #22xx): bump last_sync_at as a heartbeat on every successful
// 0-changes sync. D4 invariant ("never advance last_commit on partial") is
// preserved: last_sync_at is a monitoring signal (doctor sync_freshness
@@ -2176,6 +2348,14 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
[opts.sourceId],
);
}
// First post-upgrade sync on a pre-v125 brain lands here with
// configNeverStamped=true; stamp the fingerprint so the gate has a
// baseline for the NEXT pass. A spurious re-walk on the upgrade
// pass would surprise users; quietly establishing the baseline does
// not.
if (configNeverStamped) {
await writeConfigFingerprint(engine, opts.sourceId, currentConfigFp);
}
return {
status: 'up_to_date',
fromCommit: lastCommit,
@@ -2187,13 +2367,21 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
};
}
if ((versionMismatch || versionNeverSet) && lastCommit === headCommit) {
if ((versionMismatch || versionNeverSet || configMismatch) && lastCommit === headCommit) {
const reasons: string[] = [];
if (versionMismatch || versionNeverSet) {
reasons.push(`chunker_version=${storedVersion ?? 'unset'}${currentVersion}`);
}
if (configMismatch) {
reasons.push(`config_fingerprint=${storedConfigFp?.slice(0, 8)}${currentConfigFp.slice(0, 8)}`);
}
slog(
`[sync] chunker_version gate: stored=${storedVersion ?? 'unset'}, current=${currentVersion}. ` +
`Forcing full re-chunk pass (git HEAD unchanged but pipeline version advanced).`,
`[sync] full re-walk forced (${reasons.join(', ')}): ` +
`git HEAD unchanged but a walk-affecting setting advanced.`,
);
const result = await performFullSync(engine, fullSyncRoots, headCommit, opts);
await writeChunkerVersion(engine, opts.sourceId, currentVersion);
await writeConfigFingerprint(engine, opts.sourceId, currentConfigFp);
return result;
}
@@ -2237,8 +2425,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
scoped && p.startsWith(syncScopeRelPath + '/') ? p.slice(syncScopeRelPath.length + 1) : p;
const excluded = (p: string): boolean =>
opts.exclude !== undefined && opts.exclude.length > 0 && matchesAnyGlob(scopeRel(p), opts.exclude);
// #2156: include globs are an allow-list, same scope-relative anchoring as
// exclude. Populated from the source row's persisted config.include_globs
// (or CLI --include). Deliberately NOT threaded into syncOpts/isSyncable:
// the unsyncable-cleanup loop below deletes pages for non-metafile
// classifications, and glob filtering must stay conservative (never delete
// previously-imported pages — the documented #1433 posture for --exclude).
const included = (p: string): boolean =>
opts.include === undefined || opts.include.length === 0 || matchesAnyGlob(scopeRel(p), opts.include);
// Filter to syncable files (strategy-aware + scope-aware + exclude-aware)
// Filter to syncable files (strategy-aware + scope-aware + glob-aware)
const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined;
// #1970 (F-C): a rename whose DESTINATION is unsyncable drops out of BOTH
// `renamed` (only `r.to` is kept below) AND `deleted` (git emits it as `R`,
@@ -2252,13 +2448,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
!(inScope(r.to) && isSyncable(r.to, syncOpts)))
.map(r => r.from);
const filtered: SyncManifest = {
added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
added: manifest.added.filter(p => inScope(p) && included(p) && !excluded(p) && isSyncable(p, syncOpts)),
modified: manifest.modified.filter(p => inScope(p) && included(p) && !excluded(p) && isSyncable(p, syncOpts)),
deleted: unique([
...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)),
...renamedToUnsyncable,
]),
renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
renamed: manifest.renamed.filter(r => inScope(r.to) && included(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
};
// NAV-4: warn when --exclude filtered out every candidate change — almost
@@ -2355,6 +2551,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(gitContextRoot, pin));
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
await writeConfigFingerprint(engine, opts.sourceId, currentConfigFp);
await clearOpCheckpoint(engine, ckpt.paths);
await clearOpCheckpoint(engine, ckpt.target);
return {
@@ -3179,6 +3376,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
await writeConfigFingerprint(engine, opts.sourceId, currentConfigFp);
await clearOpCheckpoint(engine, ckpt.paths);
await clearOpCheckpoint(engine, ckpt.target);
};
@@ -3430,6 +3628,9 @@ async function performFullSync(
// files were waiting.
if (opts.dryRun) {
let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' });
if (opts.include && opts.include.length > 0) {
allFiles = allFiles.filter(abs => matchesAnyGlob(relative(syncScopeRoot, abs), opts.include));
}
if (opts.exclude && opts.exclude.length > 0) {
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
}
@@ -3475,6 +3676,7 @@ async function performFullSync(
commit: headCommit,
strategy: opts.strategy,
sourceId: opts.sourceId,
include: opts.include,
exclude: opts.exclude,
slugRoot,
// issue #1939: performFullSync owns the failure ledger + bookmark via the
@@ -3504,6 +3706,7 @@ async function performFullSync(
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
await stampSourceConfigFingerprint(engine, opts.sourceId);
};
const fullGate = await applySyncFailureGate({
@@ -3967,8 +4170,12 @@ Options:
run at the repo root; imports are scoped to the subdir
and slugs stay root-relative (wiki/page1). Passing the
subdirectory directly as --repo also works.
--include <glob> Only sync files matching at least one glob (repeatable;
matched against the scope-relative path). Merged with
the source's persisted config.include_globs.
--exclude <glob> Exclude files matching the glob from sync (repeatable;
matched against the scope-relative path).
matched against the scope-relative path; applied after
--include). Merged with config.exclude_globs.
--dry-run Show what would be synced without writing.
--skip-failed Acknowledge previously-recorded sync failures so
the bookmark can advance past unparseable files.
@@ -4129,14 +4336,17 @@ See also:
}
const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined;
// #753/#774: monorepo subdir-source flags. --exclude is repeatable.
// #2156: --include is the allow-list counterpart, same repeatable shape.
const srcSubpath = args.find((a, i) => args[i - 1] === '--src-subpath') || undefined;
const excludePatterns: string[] = [];
const includePatterns: string[] = [];
for (let i = 0; i < args.length; i++) {
if (args[i] === '--exclude' && i + 1 < args.length) excludePatterns.push(args[i + 1]);
if (args[i] === '--include' && i + 1 < args.length) includePatterns.push(args[i + 1]);
}
if (syncAll && (srcSubpath || excludePatterns.length > 0)) {
if (syncAll && (srcSubpath || excludePatterns.length > 0 || includePatterns.length > 0)) {
console.error(
`--src-subpath/--exclude scope a single sync invocation; they cannot be combined with --all. ` +
`--src-subpath/--include/--exclude scope a single sync invocation; they cannot be combined with --all. ` +
`For --all runs, register the subdirectory as the source's local_path instead ` +
`(gbrain sources add <id> --path <repo>/<subdir>).`,
);
@@ -4337,7 +4547,11 @@ See also:
const onAllSigint = () => { try { allInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
const runOne = async (src: typeof sources[number]): Promise<SyncResult> => {
const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' };
const cfg = (src.config || {}) as {
strategy?: 'markdown' | 'code' | 'auto';
include_globs?: unknown;
exclude_globs?: unknown;
};
// D18: parallel path defers embed; auto-enqueue embed-backfill after.
// v0.42.42.0 (#2139): `autoDeferEmbeds` (the inline gate tripped in a
// non-TTY session) ALSO forces deferral — global by design (the gate's
@@ -4375,6 +4589,8 @@ See also:
skipFailed, retryFailed, noSchemaPack,
sourceId: src.id,
strategy: cfg.strategy,
include: parseGlobList(cfg.include_globs),
exclude: parseGlobList(cfg.exclude_globs),
concurrency,
signal: composeAbortSignals(allInterrupt.signal, controller?.signal),
};
@@ -4586,11 +4802,27 @@ See also:
// lock released by its own finally) instead of a hard cut.
const singleSourceInterrupt = new AbortController();
const onSingleSourceSigint = () => { try { singleSourceInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
// Read persisted include/exclude globs from the source row, mirroring the
// --all fan-out's `runOne` closure above. Best-effort: a fetch failure
// falls through to "no glob filters", preserving pre-existing behavior.
// sourceId is always set here (resolveSourceWithTier ran above), so this
// path never silently runs without source-config awareness.
let sourceCfg: { include_globs?: unknown; exclude_globs?: unknown } = {};
try {
const { fetchSource } = await import('../core/sources-load.ts');
const src = await fetchSource(engine, sourceId);
if (src?.config && typeof src.config === 'object') {
sourceCfg = src.config as { include_globs?: unknown; exclude_globs?: unknown };
}
} catch { /* fall through to no filters */ }
const opts: SyncOpts = {
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId,
strategy: strategyArg, concurrency,
srcSubpath,
exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
// #2156: union of the repeatable CLI flags (one-off, this invocation
// only) and the source row's persisted config globs (every sync).
include: mergeGlobs(includePatterns, parseGlobList(sourceCfg.include_globs)),
exclude: mergeGlobs(excludePatterns, parseGlobList(sourceCfg.exclude_globs)),
signal: composeAbortSignals(singleSourceInterrupt.signal, singleSourceController?.signal),
};
@@ -4818,7 +5050,11 @@ export async function syncOneSource(
noExtract?: boolean;
},
): Promise<{ result: SyncResult; log: string }> {
const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' };
const cfg = (src.config || {}) as {
strategy?: 'markdown' | 'code' | 'auto';
include_globs?: unknown;
exclude_globs?: unknown;
};
const log = `\n--- Syncing source: ${src.name} ---\n`;
const repoOpts: SyncOpts = {
repoPath: src.local_path!,
@@ -4832,6 +5068,8 @@ export async function syncOneSource(
noSchemaPack: shared.noSchemaPack,
sourceId: src.id,
strategy: cfg.strategy,
include: parseGlobList(cfg.include_globs),
exclude: parseGlobList(cfg.exclude_globs),
concurrency: shared.concurrency,
// lockId defaults to `gbrain-sync:${src.id}` via the invariant in
// performSync (no explicit override needed — sourceId triggers it).
+26
View File
@@ -5671,6 +5671,32 @@ export const MIGRATIONS: Migration[] = [
`);
},
},
{
version: 125,
name: 'sources_config_fingerprint',
// #2157 follow-on: the "Already up to date" gate at sync.ts honors
// git-HEAD equality + chunker-version match but ignored source-config
// drift. A user who runs `gbrain sources add default --exclude
// 'Templates/**'` AFTER an initial sync got "Already up to date" on
// the next pass because git HEAD was unchanged — the new exclusion
// never reached the walk until `gbrain sync --full`.
//
// This column caches a SHA-256 fingerprint of the walk-affecting
// fields in `sources.config` (strategy + include_globs +
// exclude_globs); mismatches trigger a full re-walk via the same code
// path as a chunker_version bump.
//
// NULL on pre-migration rows is treated as "not yet stamped" by
// readConfigFingerprint, so the FIRST sync after upgrade is normal
// (no spurious force-full just because the column was added).
//
// Keep in sync with src/schema.sql and src/core/schema-embedded.ts.
idempotent: true,
sql: `
ALTER TABLE sources
ADD COLUMN IF NOT EXISTS config_fingerprint TEXT;
`,
},
];
export const LATEST_VERSION = MIGRATIONS.length > 0
+1 -1
View File
@@ -55,7 +55,7 @@ export interface AgentClientBindings {
* `redirect_uri` containing `,`) would be parsed by Postgres as MULTIPLE
* array elements, smuggling values past validation. See CSO finding #5.
*/
export function pgArray(arr: string[]): string {
function pgArray(arr: string[]): string {
if (!arr || arr.length === 0) return '{}';
const escaped = arr.map(s => `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`);
return `{${escaped.join(',')}}`;
+7
View File
@@ -39,6 +39,13 @@ CREATE TABLE IF NOT EXISTS sources (
-- bypassing the git-HEAD up_to_date early-return so CHUNKER_VERSION bumps
-- actually trigger re-chunking on upgrade.
chunker_version TEXT,
-- #2157 follow-on: SHA-256 fingerprint of the walk-affecting fields in
-- \`config\` (strategy + include_globs + exclude_globs). Mismatch forces a
-- full re-walk via the same code path as chunker_version, so a user who
-- changes \`sources.config.exclude_globs\` mid-life doesn't get "Already up
-- to date" on the next sync. NULL on pre-migration rows is treated as
-- "not yet stamped" and skips the gate (preserves first-run semantics).
config_fingerprint TEXT,
-- v0.26.5: soft-delete + recovery window. \`archive\` flips archived=true and
-- sets archive_expires_at = now() + 72h. The autopilot purge phase
-- hard-deletes rows where archive_expires_at <= now(). Promoted from a
+25
View File
@@ -155,6 +155,19 @@ export interface AddSourceOpts {
* runs). Does NOT auto-`git init` anything see `addSource` docstring.
*/
force?: boolean;
/**
* Glob filters persisted into `sources.config.include_globs` /
* `sources.config.exclude_globs`. Read at sync time by
* `commands/sync.ts:syncOneSource` and the single-source path, threaded
* into `isSyncable` / `unsyncableReason` (their `SyncableOptions` shape
* has carried this contract since v0.41.13).
*
* Empty / unspecified arrays are not persisted at all (no `[]` written
* to the JSONB), which keeps the row identical to today for sources
* that don't use filtering.
*/
includeGlobs?: string[];
excludeGlobs?: string[];
}
export interface RemoveSourceOpts {
@@ -429,6 +442,12 @@ export async function addSource(
if (opts.federated !== null && opts.federated !== undefined) {
config.federated = opts.federated;
}
if (opts.includeGlobs && opts.includeGlobs.length > 0) {
config.include_globs = opts.includeGlobs;
}
if (opts.excludeGlobs && opts.excludeGlobs.length > 0) {
config.exclude_globs = opts.excludeGlobs;
}
const displayName = opts.name ?? opts.id;
try {
@@ -508,6 +527,12 @@ export async function addSource(
if (opts.federated !== null && opts.federated !== undefined) {
config.federated = opts.federated;
}
if (opts.includeGlobs && opts.includeGlobs.length > 0) {
config.include_globs = opts.includeGlobs;
}
if (opts.excludeGlobs && opts.excludeGlobs.length > 0) {
config.exclude_globs = opts.excludeGlobs;
}
const displayName = opts.name ?? opts.id;
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config)
+7
View File
@@ -219,6 +219,13 @@ function globToRegex(pattern: string): RegExp {
return new RegExp(regex);
}
/**
* Test a normalized POSIX-style path against an array of glob patterns. Returns
* true if any pattern matches. Empty / undefined `patterns` returns false (no
* filter engaged). Exported so non-sync surfaces (lint walker, future ingest
* variants) can apply the same glob semantics as `isSyncable` without
* re-declaring `globToRegex`.
*/
export function matchesAnyGlob(path: string, patterns?: string[]): boolean {
if (!patterns || patterns.length === 0) return false;
const normalized = path.replace(/\\/g, '/');
+7
View File
@@ -35,6 +35,13 @@ CREATE TABLE IF NOT EXISTS sources (
-- bypassing the git-HEAD up_to_date early-return so CHUNKER_VERSION bumps
-- actually trigger re-chunking on upgrade.
chunker_version TEXT,
-- #2157 follow-on: SHA-256 fingerprint of the walk-affecting fields in
-- `config` (strategy + include_globs + exclude_globs). Mismatch forces a
-- full re-walk via the same code path as chunker_version, so a user who
-- changes `sources.config.exclude_globs` mid-life doesn't get "Already up
-- to date" on the next sync. NULL on pre-migration rows is treated as
-- "not yet stamped" and skips the gate (preserves first-run semantics).
config_fingerprint TEXT,
-- v0.26.5: soft-delete + recovery window. `archive` flips archived=true and
-- sets archive_expires_at = now() + 72h. The autopilot purge phase
-- hard-deletes rows where archive_expires_at <= now(). Promoted from a
-82
View File
@@ -1,82 +0,0 @@
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { queryAdminSources } from '../src/commands/serve-http.ts';
import { buildSyncStatusReport } from '../src/commands/sync.ts';
/**
* v0.41.29 Sources tab `/admin/api/sources` endpoint SQL.
*
* The endpoint is a thin Express handler over `queryAdminSources` +
* `buildSyncStatusReport`; the source-selection SQL is the load-bearing
* surface (same pattern as test/admin-agents-spend.test.ts).
*
* Pinned behaviors:
* - Excludes archived sources
* - INCLUDES sources with null local_path (push-only brains: filtering
* on local_path emptied the Sources tab + federation source-picker)
* - JSONB config surfaces as an object, defaulting to {}
* - Deterministic ORDER BY id
* - buildSyncStatusReport accepts the rows (no disk I/O on null paths)
*/
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
});
describe('queryAdminSources (/admin/api/sources SQL)', () => {
it('includes push-only sources with null local_path', async () => {
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config)
VALUES ('push-only', 'push-only', NULL, '{}'::jsonb)`,
);
const sources = await queryAdminSources(engine);
const ids = sources.map((s) => s.id);
expect(ids).toContain('push-only');
expect(sources.find((s) => s.id === 'push-only')!.local_path).toBe(null);
});
it('excludes archived sources', async () => {
await engine.executeRaw(
`INSERT INTO sources (id, name, archived) VALUES ('gone', 'gone', true)`,
);
const sources = await queryAdminSources(engine);
expect(sources.map((s) => s.id)).not.toContain('gone');
});
it('surfaces JSONB config as an object and orders by id', async () => {
await engine.executeRaw(
`INSERT INTO sources (id, name, config)
VALUES ('bbb', 'bbb', '{"syncEnabled": true}'::jsonb),
('aaa', 'aaa', '{}'::jsonb)`,
);
const sources = await queryAdminSources(engine);
const ids = sources.map((s) => s.id);
expect(ids.indexOf('aaa')).toBeLessThan(ids.indexOf('bbb'));
expect(sources.find((s) => s.id === 'bbb')!.config).toEqual({ syncEnabled: true });
expect(sources.find((s) => s.id === 'aaa')!.config).toEqual({});
});
it('buildSyncStatusReport accepts the rows (null local_path does not throw)', async () => {
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config)
VALUES ('push-only', 'push-only', NULL, '{}'::jsonb)`,
);
const report = await buildSyncStatusReport(engine, await queryAdminSources(engine));
expect(report.schema_version).toBe(1);
const row = report.sources.find((s) => s.source_id === 'push-only');
expect(row).toBeDefined();
});
});
-568
View File
@@ -1,568 +0,0 @@
/**
* Tests for `gbrain auth grant-read|revoke-read|set-federated-read`.
*
* Pure helper: parseSourceCsv (no DB).
* DB-coupled: resolveClient, assertSourceExists, *Core fns exercised
* against a real PGLite via the canonical block.
*/
import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { sqlQueryForEngine } from '../src/core/sql-query.ts';
import { pgArray } from '../src/core/oauth-provider.ts';
import {
parseSourceCsv,
resolveClient,
assertSourceExists,
grantReadCore,
revokeReadCore,
setFederatedReadCore,
extractDryRun,
sanitizeForTerminal,
} from '../src/commands/auth.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
});
// ---------------------------------------------------------------------------
// pure helpers
// ---------------------------------------------------------------------------
describe('parseSourceCsv', () => {
test('splits and trims', () => {
expect(parseSourceCsv('a,b,c')).toEqual(['a', 'b', 'c']);
expect(parseSourceCsv(' a , b ')).toEqual(['a', 'b']);
});
test('drops empty segments', () => {
expect(parseSourceCsv('a,,b,')).toEqual(['a', 'b']);
expect(parseSourceCsv(',,')).toEqual([]);
expect(parseSourceCsv('')).toEqual([]);
});
test('dedupes while preserving first-seen order', () => {
expect(parseSourceCsv('a,b,a,c,b')).toEqual(['a', 'b', 'c']);
});
});
describe('extractDryRun', () => {
test('absent flag → false', () => {
expect(extractDryRun(['alice', 'proj-x'])).toEqual({
dryRun: false,
rest: ['alice', 'proj-x'],
});
});
test('flag at end', () => {
expect(extractDryRun(['alice', 'proj-x', '--dry-run'])).toEqual({
dryRun: true,
rest: ['alice', 'proj-x'],
});
});
test('flag at start', () => {
expect(extractDryRun(['--dry-run', 'alice', 'proj-x'])).toEqual({
dryRun: true,
rest: ['alice', 'proj-x'],
});
});
test('flag in middle', () => {
expect(extractDryRun(['alice', '--dry-run', 'proj-x'])).toEqual({
dryRun: true,
rest: ['alice', 'proj-x'],
});
});
test('no args', () => {
expect(extractDryRun([])).toEqual({ dryRun: false, rest: [] });
});
});
// ---------------------------------------------------------------------------
// DB-coupled
// ---------------------------------------------------------------------------
async function seedSource(id: string): Promise<void> {
const sql = sqlQueryForEngine(engine);
await sql`INSERT INTO sources (id, name) VALUES (${id}, ${id}) ON CONFLICT (id) DO NOTHING`;
}
async function seedClient(name: string, federated: string[] = []): Promise<string> {
// Ensure write source FK is satisfied — every seeded client points at 'default'.
await seedSource('default');
const sql = sqlQueryForEngine(engine);
const clientId = `gbrain_cl_test_${name}_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`;
const fedLit = pgArray(federated);
await sql`
INSERT INTO oauth_clients (client_id, client_name, client_secret_hash,
redirect_uris, grant_types, scope,
client_id_issued_at, source_id, federated_read)
VALUES (${clientId}, ${name}, ${'dummy-hash'},
${pgArray([])}, ${pgArray(['client_credentials'])}, ${'read'},
${Date.now()}, ${'default'}, ${fedLit})
`;
return clientId;
}
async function readFederated(clientId: string): Promise<string[]> {
const sql = sqlQueryForEngine(engine);
const rows = await sql`SELECT federated_read FROM oauth_clients WHERE client_id = ${clientId}`;
const fed = rows[0]?.federated_read;
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
}
describe('resolveClient', () => {
test('matches by client_id', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const c = await resolveClient(sql, id);
expect(c.client_name).toBe('alice');
expect(c.federated_read).toEqual(['default']);
});
test('matches by client_name', async () => {
await seedSource('default');
await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const c = await resolveClient(sql, 'alice');
expect(c.client_name).toBe('alice');
});
test('errors loudly on no-match', async () => {
const sql = sqlQueryForEngine(engine);
await expect(resolveClient(sql, 'nobody')).rejects.toThrow(/No active OAuth client found/);
});
test('errors loudly on ambiguous client_name', async () => {
await seedSource('default');
await seedClient('bob', ['default']);
await seedClient('bob', ['default']);
const sql = sqlQueryForEngine(engine);
await expect(resolveClient(sql, 'bob')).rejects.toThrow(/Multiple active OAuth clients named/);
});
test('null source_id is preserved as null (legacy row tolerance)', async () => {
const sql = sqlQueryForEngine(engine);
const clientId = `gbrain_cl_test_null_${Date.now()}`;
await sql`
INSERT INTO oauth_clients (client_id, client_name, client_secret_hash,
redirect_uris, grant_types, scope,
client_id_issued_at, source_id, federated_read)
VALUES (${clientId}, ${'legacy'}, ${'dummy'},
${pgArray([])}, ${pgArray(['client_credentials'])}, ${'read'},
${Date.now()}, ${null}, ${pgArray([])})
`;
const c = await resolveClient(sql, clientId);
expect(c.source_id).toBeNull();
expect(c.federated_read).toEqual([]);
});
});
describe('assertSourceExists', () => {
test('passes when present', async () => {
await seedSource('proj-x');
const sql = sqlQueryForEngine(engine);
await expect(assertSourceExists(sql, 'proj-x')).resolves.toBeUndefined();
});
test('throws with paste-ready hint when missing', async () => {
const sql = sqlQueryForEngine(engine);
await expect(assertSourceExists(sql, 'ghost')).rejects.toThrow(
/Source "ghost" does not exist.*gbrain sources add ghost/s,
);
});
});
describe('grantReadCore', () => {
test('appends when not present and persists', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
expect(outcome.kind).toBe('updated');
if (outcome.kind === 'updated') {
expect(outcome.before).toEqual(['default']);
expect(outcome.after).toEqual(['default', 'proj-x']);
}
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
});
test('is idempotent — second call is a noop, list unchanged', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
await grantReadCore(sql, 'alice', 'proj-x');
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
expect(outcome.kind).toBe('noop');
if (outcome.kind === 'noop') {
expect(outcome.reason).toBe('already-granted');
}
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
});
test('refuses unknown source (fails BEFORE mutating)', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
await expect(grantReadCore(sql, 'alice', 'ghost')).rejects.toThrow(/does not exist/);
expect(await readFederated(id)).toEqual(['default']);
});
test('refuses unknown client', async () => {
const sql = sqlQueryForEngine(engine);
await expect(grantReadCore(sql, 'nobody', 'whatever')).rejects.toThrow(/No active OAuth client found/);
});
test('accepts client_id resolution too', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
await grantReadCore(sql, id, 'proj-x');
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
});
test('rejects malformed source_id BEFORE existence check (Codex finding #3)', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
// Even with a row in `sources` having a weird id, the validator at the
// boundary refuses. Closes the "manual SQL plants a row, CLI lets it
// become unmanageable in federated_read" vector.
await sql`INSERT INTO sources (id, name) VALUES (${'has,"weird"-bits'}, ${'weird'})`;
await expect(grantReadCore(sql, 'alice', 'has,"weird"-bits')).rejects.toThrow(/Invalid source_id/);
// DB unchanged.
expect(await readFederated(id)).toEqual(['default']);
});
});
describe('revokeReadCore', () => {
test('removes when present', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default', 'proj-x']);
const sql = sqlQueryForEngine(engine);
const outcome = await revokeReadCore(sql, 'alice', 'proj-x');
expect(outcome.kind).toBe('updated');
expect(await readFederated(id)).toEqual(['default']);
});
test('is idempotent — second call is a noop, list unchanged', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const outcome = await revokeReadCore(sql, 'alice', 'ghost-source');
expect(outcome.kind).toBe('noop');
if (outcome.kind === 'noop') {
expect(outcome.reason).toBe('not-present');
}
expect(await readFederated(id)).toEqual(['default']);
});
test('allows clearing the list down to empty (no implicit guard)', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
await revokeReadCore(sql, 'alice', 'default');
expect(await readFederated(id)).toEqual([]);
});
test('does NOT validate the source exists — operator may revoke stale references', async () => {
await seedSource('default');
// federated_read carries 'proj-x' but the source row was deleted.
const id = await seedClient('alice', ['default', 'proj-x']);
const sql = sqlQueryForEngine(engine);
const outcome = await revokeReadCore(sql, 'alice', 'proj-x');
expect(outcome.kind).toBe('updated');
expect(await readFederated(id)).toEqual(['default']);
});
});
describe('setFederatedReadCore', () => {
test('replaces list wholesale', async () => {
await seedSource('a');
await seedSource('b');
await seedSource('c');
const id = await seedClient('alice', ['a']);
const sql = sqlQueryForEngine(engine);
const outcome = await setFederatedReadCore(sql, 'alice', 'b,c');
expect(outcome.kind).toBe('updated');
expect(await readFederated(id)).toEqual(['b', 'c']);
});
test('dedupes CSV input', async () => {
await seedSource('a');
await seedSource('b');
const id = await seedClient('alice', []);
const sql = sqlQueryForEngine(engine);
await setFederatedReadCore(sql, 'alice', 'a,b,a,b,a');
expect(await readFederated(id)).toEqual(['a', 'b']);
});
test('empty string clears the list', async () => {
await seedSource('a');
const id = await seedClient('alice', ['a']);
const sql = sqlQueryForEngine(engine);
await setFederatedReadCore(sql, 'alice', '');
expect(await readFederated(id)).toEqual([]);
});
test('noop when result equals current list', async () => {
await seedSource('a');
await seedSource('b');
const id = await seedClient('alice', ['a', 'b']);
const sql = sqlQueryForEngine(engine);
const outcome = await setFederatedReadCore(sql, 'alice', 'a,b');
expect(outcome.kind).toBe('noop');
if (outcome.kind === 'noop') {
expect(outcome.reason).toBe('same-list');
}
expect(await readFederated(id)).toEqual(['a', 'b']);
});
test('refuses unknown source (fails BEFORE mutating)', async () => {
await seedSource('a');
const id = await seedClient('alice', ['a']);
const sql = sqlQueryForEngine(engine);
await expect(setFederatedReadCore(sql, 'alice', 'a,ghost')).rejects.toThrow(/does not exist/);
// Original list preserved.
expect(await readFederated(id)).toEqual(['a']);
});
test('order in CSV is the order persisted', async () => {
await seedSource('a');
await seedSource('b');
await seedSource('c');
const id = await seedClient('alice', ['a']);
const sql = sqlQueryForEngine(engine);
await setFederatedReadCore(sql, 'alice', 'c,a,b');
expect(await readFederated(id)).toEqual(['c', 'a', 'b']);
});
});
// ---------------------------------------------------------------------------
// --dry-run semantics
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
// Codex fixes: soft-delete filter, atomic-SQL race-safety, sanitizer
// ---------------------------------------------------------------------------
describe('sanitizeForTerminal', () => {
test('preserves printable ASCII unchanged', () => {
expect(sanitizeForTerminal('alice')).toBe('alice');
expect(sanitizeForTerminal('a b-c_d.e/f@g')).toBe('a b-c_d.e/f@g');
});
test('escapes ANSI escape sequences', () => {
expect(sanitizeForTerminal('\x1b[2J')).toBe('\\x1b[2J');
expect(sanitizeForTerminal('\x1b]0;TITLE\x07')).toBe('\\x1b]0;TITLE\\x07');
});
test('escapes ALL C0 controls including tab and newline', () => {
// Codex re-review: preserving \n lets a DCR-registered name spoof
// additional rows in list-clients output. Tab spoofs field separators.
// Both are now escaped.
expect(sanitizeForTerminal('\x00\x07\x08')).toBe('\\x00\\x07\\x08');
expect(sanitizeForTerminal('line1\nline2')).toBe('line1\\x0aline2');
expect(sanitizeForTerminal('col1\tcol2')).toBe('col1\\x09col2');
});
test('escapes DEL and C1 controls', () => {
expect(sanitizeForTerminal('\x7f')).toBe('\\x7f');
expect(sanitizeForTerminal('\x9b[31m')).toBe('\\x9b[31m');
});
test('passes through unicode', () => {
expect(sanitizeForTerminal('café')).toBe('café');
expect(sanitizeForTerminal('日本語')).toBe('日本語');
});
});
describe('soft-delete filter (Codex finding #2)', () => {
async function softDeleteClient(clientId: string): Promise<void> {
const sql = sqlQueryForEngine(engine);
await sql`UPDATE oauth_clients SET deleted_at = now() WHERE client_id = ${clientId}`;
}
test('resolveClient hides soft-deleted clients by default', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
await softDeleteClient(id);
const sql = sqlQueryForEngine(engine);
await expect(resolveClient(sql, 'alice')).rejects.toThrow(/No active OAuth client found/);
await expect(resolveClient(sql, id)).rejects.toThrow(/No active OAuth client found/);
});
test('resolveClient with includeDeleted finds soft-deleted clients', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
await softDeleteClient(id);
const sql = sqlQueryForEngine(engine);
const c = await resolveClient(sql, id, { includeDeleted: true });
expect(c.client_name).toBe('alice');
expect(c.deleted_at).not.toBeNull();
});
test('grantReadCore refuses to mutate soft-deleted clients', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default']);
await softDeleteClient(id);
const sql = sqlQueryForEngine(engine);
await expect(grantReadCore(sql, 'alice', 'proj-x')).rejects.toThrow(/No active OAuth client found/);
expect(await readFederated(id)).toEqual(['default']);
});
test('revokeReadCore refuses to mutate soft-deleted clients', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
await softDeleteClient(id);
const sql = sqlQueryForEngine(engine);
await expect(revokeReadCore(sql, 'alice', 'default')).rejects.toThrow(/No active OAuth client found/);
expect(await readFederated(id)).toEqual(['default']);
});
test('two clients with same name but only one active resolves to the active one', async () => {
await seedSource('default');
// Seed two clients with the same name; soft-delete the older one.
const sql = sqlQueryForEngine(engine);
const oldId = await seedClient('alice', ['default']);
await softDeleteClient(oldId);
const newId = await seedClient('alice', ['default']); // same name, new row
const c = await resolveClient(sql, 'alice');
expect(c.client_id).toBe(newId); // active row wins; ambiguity error suppressed
});
});
describe('atomic SQL race-safety (Codex finding #1, HIGH)', () => {
test('grant+revoke serialize at row-lock — sensitive stays revoked', async () => {
await seedSource('default');
await seedSource('sensitive');
await seedSource('harmless');
const id = await seedClient('alice', ['default', 'sensitive']);
const sql = sqlQueryForEngine(engine);
// Simulate concurrent revoke(sensitive) + grant(harmless). Real concurrency
// would race at the JS event loop boundary; here we await sequentially but
// each call goes through the ATOMIC SQL path. The contract: regardless of
// ordering, the final state has sensitive REMOVED and harmless ADDED.
await revokeReadCore(sql, 'alice', 'sensitive');
await grantReadCore(sql, 'alice', 'harmless');
const final1 = await readFederated(id);
expect(final1.sort()).toEqual(['default', 'harmless']);
// Reverse order, same final state. The pre-fix read-modify-write shape
// would have produced ['default', 'sensitive', 'harmless'] here (the
// resurrection bug Codex caught).
const id2 = await seedClient('bob', ['default', 'sensitive']);
await grantReadCore(sql, 'bob', 'harmless');
await revokeReadCore(sql, 'bob', 'sensitive');
const final2 = await readFederated(id2);
expect(final2.sort()).toEqual(['default', 'harmless']);
});
test('grant uses RETURNING to surface the post-write state', async () => {
await seedSource('default');
await seedSource('proj-x');
await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
expect(outcome.kind).toBe('updated');
if (outcome.kind === 'updated') {
// The `after` came from RETURNING, not from computing prev+sourceId
// in JS — proves the atomic path returned authoritative state.
expect(outcome.after).toEqual(['default', 'proj-x']);
}
});
test('grant noop path still survives without writing', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const outcome = await grantReadCore(sql, 'alice', 'default');
expect(outcome.kind).toBe('noop');
if (outcome.kind === 'noop') expect(outcome.reason).toBe('already-granted');
expect(await readFederated(id)).toEqual(['default']);
});
});
describe('dryRun mode', () => {
test('grantReadCore returns "updated" outcome but skips the write', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
const outcome = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: true });
expect(outcome.kind).toBe('updated');
if (outcome.kind === 'updated') {
expect(outcome.before).toEqual(['default']);
expect(outcome.after).toEqual(['default', 'proj-x']);
}
// Crucially: the DB row is UNCHANGED.
expect(await readFederated(id)).toEqual(['default']);
});
test('revokeReadCore returns "updated" outcome but skips the write', async () => {
await seedSource('default');
await seedSource('proj-x');
const id = await seedClient('alice', ['default', 'proj-x']);
const sql = sqlQueryForEngine(engine);
const outcome = await revokeReadCore(sql, 'alice', 'proj-x', { dryRun: true });
expect(outcome.kind).toBe('updated');
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
});
test('setFederatedReadCore returns "updated" outcome but skips the write', async () => {
await seedSource('a');
await seedSource('b');
await seedSource('c');
const id = await seedClient('alice', ['a']);
const sql = sqlQueryForEngine(engine);
const outcome = await setFederatedReadCore(sql, 'alice', 'b,c', { dryRun: true });
expect(outcome.kind).toBe('updated');
expect(await readFederated(id)).toEqual(['a']);
});
test('noop outcomes are surfaced identically with or without dryRun', async () => {
await seedSource('default');
await seedSource('proj-x');
await seedClient('alice', ['default', 'proj-x']);
const sql = sqlQueryForEngine(engine);
const live = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: false });
const dry = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: true });
expect(live.kind).toBe('noop');
expect(dry.kind).toBe('noop');
});
test('errors still fire in dryRun (operator sees the problem before commit)', async () => {
await seedSource('default');
const id = await seedClient('alice', ['default']);
const sql = sqlQueryForEngine(engine);
await expect(
grantReadCore(sql, 'alice', 'ghost', { dryRun: true }),
).rejects.toThrow(/does not exist/);
await expect(
grantReadCore(sql, 'nobody', 'default', { dryRun: true }),
).rejects.toThrow(/No active OAuth client found/);
// DB unchanged.
expect(await readFederated(id)).toEqual(['default']);
});
});
-23
View File
@@ -174,29 +174,6 @@ describe('parseRegisterClientArgs', () => {
});
describe('error cases', () => {
test('--source with malformed id throws (validates source_id shape — codex re-review)', () => {
// Defense for the "register-client seeds an unmanageable
// federated_read entry" vector. assertValidSourceId fires before the
// function returns so DB never sees a row with bad source scope.
expect(() => parseRegisterClientArgs(['--source', 'has,weird,bits'])).toThrow(/Invalid source_id/);
expect(() => parseRegisterClientArgs(['--source', 'UPPER'])).toThrow(/Invalid source_id/);
expect(() => parseRegisterClientArgs(['--source', ''])).toThrow(/Invalid source_id|requires a value/);
});
test('--federated-read with any malformed id throws', () => {
// Single-item bad.
expect(() => parseRegisterClientArgs(['--federated-read', 'bad,source!'])).toThrow(/Invalid source_id/);
// Mixed valid + invalid — fails on the first bad one.
expect(() => parseRegisterClientArgs(['--federated-read', 'good,bad source'])).toThrow(/Invalid source_id/);
});
test('--source default + --federated-read default,team passes (regression — common case)', () => {
// Sanity: the canonical real-world invocation still parses cleanly.
const out = parseRegisterClientArgs(['--source', 'default', '--federated-read', 'default,team']);
expect(out.sourceId).toBe('default');
expect(out.federatedRead).toEqual(['default', 'team']);
});
test('--redirect-uri without value → throws', () => {
expect(() => parseRegisterClientArgs(['--redirect-uri'])).toThrow(/requires a value/);
});
+153
View File
@@ -0,0 +1,153 @@
/**
* `gbrain lint` source-glob filter walker integration.
*
* PR #2157 (commit cf9a3b18, `feat/sync-source-glob-filters`) wired
* `sources.config.include_globs` / `exclude_globs` into `gbrain sync` so a
* user could exclude `Resources/veriff/**` and have every subsequent sync
* honor it. The lint command walked the same source dirs blind and emitted
* findings against paths the user had already declared out of scope a
* half-finished feature.
*
* This patch extends the same persisted glob contract to lint:
* - `gbrain lint` gains `--include / --exclude` flags (parallel to sync).
* - `runLintCore` lifts `sources.config.{include,exclude}_globs` for any
* target whose absolute path matches a source row's `local_path`, so the
* cycle.lint phase + Minion lint handlers honor the same filter without
* restating it.
* - The walker in `collectPages` applies the filter using the SAME
* `matchesAnyGlob` helper sync uses, anchored at the target dir (so a
* persisted `Resources/veriff/**` glob written against the source root
* works without rewriting it as an absolute path).
*
* These tests pin the walker contract. The engine-side lift
* (`resolveSourceGlobsForTarget`) is best-effort by design (returns `{}` on
* any error) and is exercised by the dream-cycle lint phase end-to-end.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
// runLintCore is the library entry — the same surface the cycle.lint phase
// and Minion handlers call. Exercising it covers the walker via its real
// callsite; testing `collectPages` directly would skip the wiring.
import { runLintCore } from '../src/commands/lint.ts';
// A self-contained content-sanity stub so the test never touches a real
// engine / config file. Empty operator-literal list keeps the content-sanity
// pass silent so the only findings come from the structural rules
// (no-frontmatter etc.).
const STUB_CS = {
fail_on_throw: false,
warn_on_throw: false,
bytes_warn: 1024 * 1024,
operator_literals: [],
};
describe('runLintCore — source-glob walker filter', () => {
let root: string;
beforeAll(() => {
root = mkdtempSync(join(tmpdir(), 'gbrain-lint-globs-'));
// Three subtrees with mixed structured / archive-style content.
// All pages have `# Title` headers but no frontmatter so each one
// emits at least one `no-frontmatter` issue under the default rule set.
mkdirSync(join(root, 'Notes'), { recursive: true });
mkdirSync(join(root, 'Resources', 'veriff'), { recursive: true });
mkdirSync(join(root, 'Resources', 'prior-art', 'archive-v1'), { recursive: true });
writeFileSync(join(root, 'Notes', 'a.md'), '# A\nbody\n');
writeFileSync(join(root, 'Notes', 'b.md'), '# B\nbody\n');
writeFileSync(join(root, 'Resources', 'veriff', 'spec-1.md'), '# Veriff spec 1\nbody\n');
writeFileSync(join(root, 'Resources', 'veriff', 'spec-2.md'), '# Veriff spec 2\nbody\n');
writeFileSync(join(root, 'Resources', 'prior-art', 'archive-v1', 'old.md'), '# Old\nbody\n');
});
afterAll(() => {
rmSync(root, { recursive: true, force: true });
});
test('no filter — walks every .md (regression guard for default behavior)', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
});
expect(result.pages_scanned).toBe(5);
expect(result.pages_with_issues).toBeGreaterThan(0);
});
test('exclude glob skips matching paths (Resources/veriff/** off-limits)', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
exclude: ['Resources/veriff/**'],
});
// 5 total minus 2 veriff specs = 3 pages walked.
expect(result.pages_scanned).toBe(3);
});
test('exclude with multiple patterns is union (veriff + prior-art both skipped)', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
exclude: ['Resources/veriff/**', 'Resources/prior-art/**'],
});
// 5 total minus 3 (2 veriff + 1 archive-v1) = 2 pages walked.
expect(result.pages_scanned).toBe(2);
});
test('include glob narrows the walk to matching paths only', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
include: ['Notes/**'],
});
expect(result.pages_scanned).toBe(2);
});
test('exclude runs AFTER include (same precedence as `gbrain sync`)', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
include: ['**/*.md'],
exclude: ['Resources/**'],
});
// include lets everything through; exclude drops the 3 Resources/* files.
expect(result.pages_scanned).toBe(2);
});
test('empty include / exclude arrays do NOT engage the filter', async () => {
// Symmetric with `parseGlobList` returning undefined for empty input —
// an empty include would otherwise classify every path as a miss and
// silently zero out the lint scope. Pin the guard at the walker level.
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
include: [],
exclude: [],
});
expect(result.pages_scanned).toBe(5);
});
test('exclude semantics match sync — `**` matches across path segments', async () => {
const result = await runLintCore({
target: root,
contentSanity: STUB_CS,
exclude: ['**/spec-*.md'],
});
// Both Veriff specs match the deep glob; Notes + archive-v1 survive.
expect(result.pages_scanned).toBe(3);
});
test('single-file target bypasses the filter (file mode is not a walk)', async () => {
// A user lints one .md explicitly: filters are a directory-walk concern,
// so the file is processed even if its name would match an exclude.
const result = await runLintCore({
target: join(root, 'Resources', 'veriff', 'spec-1.md'),
contentSanity: STUB_CS,
exclude: ['Resources/veriff/**'],
});
expect(result.pages_scanned).toBe(1);
});
});
+6
View File
@@ -694,6 +694,12 @@ const COLUMN_EXEMPTIONS = new Set<string>([
'minion_jobs.quiet_hours',
'minion_jobs.stagger_key',
'sources.chunker_version',
// #2157 follow-on (migration v125). TEXT column read by performSync's
// `Already up to date` gate; not referenced by any CREATE INDEX. Same
// upgrade-path coverage as sources.chunker_version above: fresh installs
// get it via the CREATE TABLE in src/schema.sql + schema-embedded.ts;
// pre-existing brains get it via the idempotent ALTER TABLE in v125.
'sources.config_fingerprint',
'access_tokens.permissions',
'takes.resolved_quality',
'pages.emotional_weight_recomputed_at',
+121
View File
@@ -159,6 +159,127 @@ describe('sources add', () => {
await expect(runSources(engine, ['add', 'plans', '--path', '/tmp/gstack/plans']))
.rejects.toThrow(/overlaps with existing source "gstack"/);
});
// Glob filters — TODO #3 from the brettdavies fork recon. Pre-fix, the
// `SyncableOptions` shape in `src/core/sync.ts` had been carrying
// `include` / `exclude` since v0.41.13, but commands/sync.ts:1454 never
// populated them and `sources add` had no flag to persist them — so users
// had no way to tell gbrain to skip `Templates/` in an Obsidian vault.
test('--exclude persists glob into sources.config.exclude_globs', async () => {
const { engine, calls } = makeStub({
'SELECT id, name, local_path, last_commit, last_sync_at, config, created_at': [{
id: 'vault',
name: 'vault',
local_path: '/tmp/vault',
last_commit: null,
last_sync_at: null,
config: '{"exclude_globs":["Templates/**"]}',
created_at: new Date(),
}],
});
await runSources(engine, ['add', 'vault', '--path', '/tmp/vault', '--exclude', 'Templates/**']);
const insert = calls.find(c => c.sql.includes('INSERT INTO sources'));
expect(insert!.params[3]).toBe('{"exclude_globs":["Templates/**"]}');
});
test('--include persists glob into sources.config.include_globs', async () => {
const { engine, calls } = makeStub({
'SELECT id, name, local_path, last_commit, last_sync_at, config, created_at': [{
id: 'wiki',
name: 'wiki',
local_path: '/tmp/wiki',
last_commit: null,
last_sync_at: null,
config: '{"include_globs":["people/**"]}',
created_at: new Date(),
}],
});
await runSources(engine, ['add', 'wiki', '--path', '/tmp/wiki', '--include', 'people/**']);
const insert = calls.find(c => c.sql.includes('INSERT INTO sources'));
expect(insert!.params[3]).toBe('{"include_globs":["people/**"]}');
});
test('--exclude is repeatable; preserves order', async () => {
const { engine, calls } = makeStub({
'SELECT id, name, local_path, last_commit, last_sync_at, config, created_at': [{
id: 'vault',
name: 'vault',
local_path: '/tmp/vault',
last_commit: null,
last_sync_at: null,
config: '{}',
created_at: new Date(),
}],
});
await runSources(engine, [
'add', 'vault', '--path', '/tmp/vault',
'--exclude', 'Templates/**',
'--exclude', '.smart-env/**',
'--exclude', 'Drafts/**',
]);
const insert = calls.find(c => c.sql.includes('INSERT INTO sources'));
expect(insert!.params[3]).toBe('{"exclude_globs":["Templates/**",".smart-env/**","Drafts/**"]}');
});
test('--include and --exclude compose in one command (federated source with both filter axes)', async () => {
const { engine, calls } = makeStub({
'SELECT id, name, local_path, last_commit, last_sync_at, config, created_at': [{
id: 'vault',
name: 'vault',
local_path: '/tmp/vault',
last_commit: null,
last_sync_at: null,
config: '{"federated":true,"include_globs":["people/**"],"exclude_globs":["Templates/**"]}',
created_at: new Date(),
}],
});
await runSources(engine, [
'add', 'vault', '--path', '/tmp/vault', '--federated',
'--include', 'people/**',
'--exclude', 'Templates/**',
]);
const insert = calls.find(c => c.sql.includes('INSERT INTO sources'));
expect(insert!.params[3]).toBe(
'{"federated":true,"include_globs":["people/**"],"exclude_globs":["Templates/**"]}',
);
});
test('omitted glob flags leave config untouched (no [] entries persisted)', async () => {
// Regression guard: empty glob arrays must NOT be written. Otherwise a
// brain that never opts into filtering grows {"include_globs": [],
// "exclude_globs": []} cruft in every source row, and the parseGlobList
// path would return undefined anyway (the cruft is purely noise).
const { engine, calls } = makeStub({
'SELECT id, name, local_path, last_commit, last_sync_at, config, created_at': [{
id: 'gstack',
name: 'gstack',
local_path: '/tmp/gstack',
last_commit: null,
last_sync_at: null,
config: '{}',
created_at: new Date(),
}],
});
await runSources(engine, ['add', 'gstack', '--path', '/tmp/gstack']);
const insert = calls.find(c => c.sql.includes('INSERT INTO sources'));
expect(insert!.params[3]).toBe('{}');
});
test('--exclude requires a glob argument', async () => {
const { engine } = makeStub();
const code = await withExitCapture(() => runSources(engine, [
'add', 'vault', '--path', '/tmp/vault', '--exclude',
]));
expect(code).toBe(2);
});
test('--include rejects a flag-like value (--include --path looks like a typo)', async () => {
const { engine } = makeStub();
const code = await withExitCapture(() => runSources(engine, [
'add', 'vault', '--path', '/tmp/vault', '--include', '--federated',
]));
expect(code).toBe(2);
});
});
// ── add — #2707 git-repo validation (CLI wiring) ───────────────
+95
View File
@@ -0,0 +1,95 @@
/**
* #2157 follow-on (migration v125) end-to-end gate wiring.
*
* `test/sync-config-fingerprint.test.ts` pins the persistence + comparison
* primitives (compute/read/write). This file pins the WIRING inside
* `performSync`: with git HEAD unchanged, a drift in the walk-affecting
* `sources.config` fields must break out of the "Already up to date" early
* return and force a full re-walk and the re-stamped fingerprint must
* settle the gate back to `up_to_date` on the following pass. Deleting the
* `configMismatch` term from the gate condition fails this test; none of the
* primitive tests would catch that.
*/
import { test, expect, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { execFileSync } from 'child_process';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { performSync } from '../src/commands/sync.ts';
let engine: PGLiteEngine;
let repoPath: string;
function git(cwd: string, ...args: string[]) {
execFileSync('git', args, { cwd, stdio: 'pipe' });
}
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-fp-gate-'));
mkdirSync(join(repoPath, 'wiki'));
mkdirSync(join(repoPath, 'memory'));
writeFileSync(join(repoPath, 'wiki', 'page1.md'), '# Page 1\n\nbody\n');
writeFileSync(join(repoPath, 'memory', 'note1.md'), '# Note 1\n\nbody\n');
git(repoPath, 'init');
git(repoPath, 'add', '-A');
git(repoPath, '-c', 'user.email=t@example.com', '-c', 'user.name=t', 'commit', '-m', 'init');
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config) VALUES ($1, $2, $3, $4::text::jsonb)`,
['vault', 'vault', repoPath, JSON.stringify({ include_globs: ['wiki/**'] })],
);
}, 60_000);
afterAll(async () => {
await engine?.disconnect();
rmSync(repoPath, { recursive: true, force: true });
});
test('config-glob drift with unchanged git HEAD forces a re-walk, then settles', async () => {
// First sync: row config include_globs = ['wiki/**'], caller threads it
// (as syncOneSource / the single-source CLI path do). memory/* skipped.
const first = await performSync(engine, {
repoPath, sourceId: 'vault', include: ['wiki/**'],
noPull: true, noEmbed: true, full: true,
});
expect(first.status).toBe('first_sync');
expect(await engine.getPage('wiki/page1')).not.toBeNull();
expect(await engine.getPage('memory/note1')).toBeNull();
// No drift, HEAD unchanged: gate stays quiet.
const second = await performSync(engine, {
repoPath, sourceId: 'vault', include: ['wiki/**'],
noPull: true, noEmbed: true,
});
expect(second.status).toBe('up_to_date');
// User widens the persisted globs (what `gbrain sources add --include`
// writes). Git HEAD has NOT moved.
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify({ include_globs: ['wiki/**', 'memory/**'] }), 'vault'],
);
// Pre-fix this returned `up_to_date` (HEAD unchanged) and memory/note1
// stayed missing until a manual `--full`. The fingerprint gate must force
// the full re-walk instead.
const third = await performSync(engine, {
repoPath, sourceId: 'vault', include: ['wiki/**', 'memory/**'],
noPull: true, noEmbed: true,
});
expect(third.status).not.toBe('up_to_date');
expect(await engine.getPage('memory/note1')).not.toBeNull();
// Re-stamped fingerprint matches the current row: gate settles.
const fourth = await performSync(engine, {
repoPath, sourceId: 'vault', include: ['wiki/**', 'memory/**'],
noPull: true, noEmbed: true,
});
expect(fourth.status).toBe('up_to_date');
});
+234
View File
@@ -0,0 +1,234 @@
/**
* #2157 follow-on (migration v125 `sources.config_fingerprint`).
*
* The "Already up to date" gate at performSync's git-HEAD equality check
* honored chunker_version match but ignored `sources.config` drift.
* Changing `sources.config.exclude_globs` (or include_globs / strategy)
* had no observable effect on the next sync because git HEAD was
* unchanged the gate returned early and the new walk scope never
* applied. This file exercises the persistence shape + drift detection
* end-to-end on PGLite, including:
*
* - Migration v125 actually adds the column (regression guard against
* a future re-numbering or accidental deletion).
* - read/write round-trips preserve the value.
* - The fingerprint differs across the three walk-affecting fields
* and is order-insensitive on the array fields.
* - NULL fingerprint on pre-v125 rows treats as "not stamped" so a
* first post-upgrade sync doesn't spuriously force-full.
* - A toggle-and-revert leaves the stored fingerprint matching the
* current row, so the gate stays quiet.
*
* The wired-up gate behavior (force-full triggered on mismatch) is
* exercised by the existing sync end-to-end tests; here we pin the
* persistence + comparison primitives the gate depends on.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import {
computeSourceConfigFingerprint,
readConfigFingerprint,
writeConfigFingerprint,
} from '../src/commands/sync.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
await engine?.disconnect();
});
/** Insert a fresh source row with the given config. Returns the id. */
async function makeSource(
id: string,
config: Record<string, unknown> = {},
): Promise<string> {
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config) VALUES ($1, $2, $3, $4::text::jsonb)`,
[id, id, `/tmp/${id}`, JSON.stringify(config)],
);
return id;
}
describe('migration v125 — sources.config_fingerprint column', () => {
test('column exists on the sources table', async () => {
const rows = await engine.executeRaw<{ column_name: string }>(
`SELECT column_name FROM information_schema.columns
WHERE table_name = 'sources' AND column_name = 'config_fingerprint'`,
);
expect(rows).toHaveLength(1);
});
test('column is nullable (preserves pre-migration row semantics)', async () => {
const rows = await engine.executeRaw<{ is_nullable: string }>(
`SELECT is_nullable FROM information_schema.columns
WHERE table_name = 'sources' AND column_name = 'config_fingerprint'`,
);
expect(rows[0]?.is_nullable).toBe('YES');
});
});
describe('readConfigFingerprint / writeConfigFingerprint — persistence round-trip', () => {
test('round-trip: write then read returns the same value', async () => {
const id = await makeSource('rt-basic', { exclude_globs: ['Templates/**'] });
const fp = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'] });
await writeConfigFingerprint(engine, id, fp);
const got = await readConfigFingerprint(engine, id);
expect(got).toBe(fp);
});
test('NULL on never-stamped row (pre-v125 semantics)', async () => {
const id = await makeSource('rt-never');
const got = await readConfigFingerprint(engine, id);
expect(got).toBeNull();
});
test('undefined sourceId returns null (legacy non-source-scoped sync)', async () => {
const got = await readConfigFingerprint(engine, undefined);
expect(got).toBeNull();
});
test('write with undefined sourceId is a no-op (does not throw)', async () => {
// The legacy global-sync code path hits this branch; the guard must
// be silent rather than fail the sync run.
await writeConfigFingerprint(engine, undefined, 'deadbeef'.repeat(8));
// No assertion beyond "did not throw"; the function returns void.
});
test('overwrite: a second write replaces the prior fingerprint', async () => {
const id = await makeSource('rt-overwrite');
await writeConfigFingerprint(engine, id, 'a'.repeat(64));
await writeConfigFingerprint(engine, id, 'b'.repeat(64));
const got = await readConfigFingerprint(engine, id);
expect(got).toBe('b'.repeat(64));
});
});
describe('end-to-end drift simulation — the gate semantics this column enables', () => {
test('first stamp matches computed fingerprint of the row config', async () => {
const cfg = { exclude_globs: ['Templates/**', 'Photos/**'], strategy: 'markdown' };
const id = await makeSource('e2e-first-stamp', cfg);
const computed = computeSourceConfigFingerprint(cfg);
await writeConfigFingerprint(engine, id, computed);
expect(await readConfigFingerprint(engine, id)).toBe(computed);
});
test('exclude_globs mutation makes stored != current (drift detected)', async () => {
const before = { exclude_globs: ['Templates/**'] };
const after = { exclude_globs: ['Templates/**', 'Photos/**'] };
const id = await makeSource('e2e-exclude-drift', before);
const beforeFp = computeSourceConfigFingerprint(before);
await writeConfigFingerprint(engine, id, beforeFp);
// Simulate the user mutating sources.config via `gbrain sources add
// --exclude`. The gate's next read of (stored, computed-from-current)
// detects the drift and forces a re-walk.
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify(after), id],
);
const afterFp = computeSourceConfigFingerprint(after);
const stored = await readConfigFingerprint(engine, id);
expect(stored).toBe(beforeFp);
expect(stored).not.toBe(afterFp);
});
test('toggle-and-revert: add then remove same pattern leaves stored matching current', async () => {
const original = { exclude_globs: ['Templates/**'] };
const id = await makeSource('e2e-toggle', original);
const originalFp = computeSourceConfigFingerprint(original);
await writeConfigFingerprint(engine, id, originalFp);
// Add a pattern (drift) then remove it (revert).
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify({ exclude_globs: ['Templates/**', 'Photos/**'] }), id],
);
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify(original), id],
);
const revertedFp = computeSourceConfigFingerprint(original);
expect(revertedFp).toBe(originalFp);
expect(await readConfigFingerprint(engine, id)).toBe(originalFp);
// ⇒ Gate compares storedFp (==originalFp) to currentFp (==originalFp): no drift, no force-full.
});
test('include_globs drift detected independently', async () => {
const before = { include_globs: ['people/**'] };
const after = { include_globs: ['people/**', 'companies/**'] };
const id = await makeSource('e2e-include-drift', before);
await writeConfigFingerprint(engine, id, computeSourceConfigFingerprint(before));
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify(after), id],
);
const stored = await readConfigFingerprint(engine, id);
const current = computeSourceConfigFingerprint(after);
expect(stored).not.toBe(current);
});
test('strategy drift detected', async () => {
const before = { strategy: 'markdown' };
const after = { strategy: 'code' };
const id = await makeSource('e2e-strategy-drift', before);
await writeConfigFingerprint(engine, id, computeSourceConfigFingerprint(before));
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[JSON.stringify(after), id],
);
expect(await readConfigFingerprint(engine, id))
.not.toBe(computeSourceConfigFingerprint(after));
});
test('mutating unrelated config field (federated) does NOT drift', async () => {
// The fingerprint hashes ONLY walk-affecting fields. Federation
// changes search visibility, not the walk set — must not invalidate
// the checkpoint.
const id = await makeSource('e2e-federated-toggle', {
federated: true,
exclude_globs: ['Templates/**'],
});
await writeConfigFingerprint(
engine,
id,
computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'] }),
);
await engine.executeRaw(
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
[
JSON.stringify({ federated: false, exclude_globs: ['Templates/**'] }),
id,
],
);
const stored = await readConfigFingerprint(engine, id);
const current = computeSourceConfigFingerprint({
federated: false,
exclude_globs: ['Templates/**'],
});
expect(stored).toBe(current);
});
test('double-encoded JSONB config (the sources-add stringify bug) hashes equivalently to the parsed object', async () => {
// `gbrain sources add` writes `JSON.stringify(config)::jsonb`, which
// double-encodes the value into a JSON-string scalar (`"{\"x\":1}"`)
// rather than a proper JSONB object. The defensive reader in
// postgres-engine.ts:1274 + readSourceConfig parses the string back
// before the fingerprint sees it, so a double-encoded row and a
// properly-shaped row must fingerprint identically.
const cfg = { exclude_globs: ['Templates/**'], strategy: 'markdown' };
const direct = computeSourceConfigFingerprint(cfg);
// The pure compute fn handles a pre-parsed object; the persistence
// layer's job is to deliver a parsed object. We assert that the
// round-trip a real read would produce (parse the string scalar)
// hashes to the same value.
const parsed = JSON.parse(JSON.stringify(cfg));
expect(computeSourceConfigFingerprint(parsed)).toBe(direct);
});
});
+63
View File
@@ -355,6 +355,69 @@ describe('sync monorepo subdir-source support (#753/#774)', () => {
expect(await engine.getPage('wiki/draft-a')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// --include: allow-list counterpart (#2156). Same scope-relative anchoring
// as --exclude; exclude applies after include.
// ─────────────────────────────────────────────────────────────────────────
test('--include: only matching files import on full sync', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const result = await performSync(engine, {
repoPath,
include: ['wiki/**'],
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(2); // wiki/page1 + wiki/page2; memory/* miss the allow-list
expect(await engine.getPage('wiki/page1')).not.toBeNull();
expect(await engine.getPage('memory/note1')).toBeNull();
});
test('--include applies to the incremental path too', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const first = await performSync(engine, {
repoPath,
include: ['wiki/**'],
noPull: true,
noEmbed: true,
full: true,
});
expect(first.status).toBe('first_sync');
writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3'));
writeFileSync(join(repoPath, 'memory', 'note3.md'), mdPage('Memory Note 3'));
gitCommit(repoPath, 'more pages');
const second = await performSync(engine, {
repoPath,
include: ['wiki/**'],
noPull: true,
noEmbed: true,
});
expect(second.status).toBe('synced');
expect(second.added).toBe(1); // wiki/page3 only; memory/note3 misses the allow-list
expect(await engine.getPage('wiki/page3')).not.toBeNull();
expect(await engine.getPage('memory/note3')).toBeNull();
});
test('--exclude applies after --include (path in both is rejected)', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const result = await performSync(engine, {
repoPath,
include: ['wiki/**'],
exclude: ['wiki/page2.md'],
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(1); // page1 only: page2 included then excluded
expect(await engine.getPage('wiki/page1')).not.toBeNull();
expect(await engine.getPage('wiki/page2')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// --exclude '**/*' emits warning (NAV-4)
// ─────────────────────────────────────────────────────────────────────────
+212
View File
@@ -0,0 +1,212 @@
/**
* TODO #3 `parseGlobList` defensive parse.
*
* `sources.config` is a JSONB column with no schema. The runtime can find
* anything in `config.include_globs` / `config.exclude_globs`:
* - A user `gbrain sources add` wrote `["people/**"]` (the happy path).
* - A stray hand-edit wrote `"people/**"` (string, not array).
* - A future migration's null default.
* - A test fixture that left the column at `{}`.
*
* The parse must produce `string[] | undefined` so the downstream
* `SyncOpts.include` / `SyncOpts.exclude` are either undefined (no filter)
* or a non-empty list of usable globs. Returning `[]` would make
* `commands/sync.ts:1454` engage the filter loop with an empty allow-list
* that classifies every path as `include-glob-miss`.
*/
import { describe, test, expect } from 'bun:test';
import { parseGlobList, mergeGlobs, computeSourceConfigFingerprint } from '../src/commands/sync.ts';
describe('parseGlobList — JSONB-safe coercion to string[] | undefined', () => {
test('happy path: array of strings round-trips identically', () => {
expect(parseGlobList(['people/**', 'companies/**'])).toEqual(['people/**', 'companies/**']);
});
test('single-element array returned as-is', () => {
expect(parseGlobList(['Templates/**'])).toEqual(['Templates/**']);
});
test('non-array values return undefined (string, object, number, null)', () => {
expect(parseGlobList('Templates/**')).toBeUndefined();
expect(parseGlobList({ globs: ['Templates/**'] })).toBeUndefined();
expect(parseGlobList(42)).toBeUndefined();
expect(parseGlobList(null)).toBeUndefined();
expect(parseGlobList(undefined)).toBeUndefined();
});
test('empty array returns undefined (no engagement of the filter loop)', () => {
// Critical: a literal `[]` must not slip through. Empty `include` in
// SyncableOptions silently passes everything (good), but empty
// `exclude` is fine too — the real motivation is to keep `SyncOpts`
// unset so callers can ignore the field entirely. Symmetric with the
// `omitted glob flags leave config untouched` regression guard in
// sources.test.ts.
expect(parseGlobList([])).toBeUndefined();
});
test('mixed array drops non-string entries and keeps the rest', () => {
expect(parseGlobList(['people/**', 42, null, 'companies/**'])).toEqual([
'people/**',
'companies/**',
]);
});
test('empty strings dropped (a `""` glob would match every path)', () => {
expect(parseGlobList(['', 'people/**', ''])).toEqual(['people/**']);
});
test('array of only empty strings collapses to undefined', () => {
expect(parseGlobList(['', '', ''])).toBeUndefined();
});
});
describe('mergeGlobs — CLI flags union with persisted source-config globs', () => {
test('both sides present: union, deduped, CLI first', () => {
expect(mergeGlobs(['a/**', 'b/**'], ['b/**', 'c/**'])).toEqual(['a/**', 'b/**', 'c/**']);
});
test('CLI only', () => {
expect(mergeGlobs(['a/**'], undefined)).toEqual(['a/**']);
});
test('persisted only', () => {
expect(mergeGlobs([], ['Templates/**'])).toEqual(['Templates/**']);
});
test('neither side: undefined so SyncOpts stays unset', () => {
expect(mergeGlobs([], undefined)).toBeUndefined();
});
});
/**
* #2157 follow-on (sources_config_fingerprint, migration v125).
*
* computeSourceConfigFingerprint hashes the walk-affecting fields of
* sources.config (strategy + include_globs + exclude_globs) so the
* "Already up to date" gate at performSync's git-HEAD equality check
* can detect drift and force a re-walk. These cases pin the contract
* the gate depends on:
*
* - Deterministic over equivalent inputs (order-insensitive,
* defensively-coerced via parseGlobList).
* - Sensitive to each walk-affecting field separately.
* - Insensitive to fields the walker doesn't read (federated,
* unrelated keys).
* - A toggle-and-revert is a no-op (returns to the original hash).
*
* Without the canonicalization the gate would fire spuriously on
* cosmetic changes (e.g. a user re-ordering their exclude list) and
* miss real drift (e.g. an add-then-remove that nets to a different
* effective set than the stored fingerprint).
*/
describe('computeSourceConfigFingerprint — walk-affecting config drift detector', () => {
test('empty config produces a stable hash', () => {
const a = computeSourceConfigFingerprint({});
const b = computeSourceConfigFingerprint({});
expect(a).toBe(b);
expect(a).toMatch(/^[a-f0-9]{64}$/);
});
test('null / undefined / missing config all hash the same', () => {
const empty = computeSourceConfigFingerprint({});
expect(computeSourceConfigFingerprint(null)).toBe(empty);
expect(computeSourceConfigFingerprint(undefined)).toBe(empty);
});
test('same config → same hash (deterministic)', () => {
const cfg = { strategy: 'markdown', exclude_globs: ['Templates/**', 'Photos/**'] };
expect(computeSourceConfigFingerprint(cfg)).toBe(computeSourceConfigFingerprint(cfg));
});
test('array order does not affect hash (canonical sort)', () => {
const a = computeSourceConfigFingerprint({ exclude_globs: ['a/**', 'b/**', 'c/**'] });
const b = computeSourceConfigFingerprint({ exclude_globs: ['c/**', 'a/**', 'b/**'] });
expect(a).toBe(b);
});
test('exclude_globs change → different hash', () => {
const a = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'] });
const b = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**', 'Photos/**'] });
expect(a).not.toBe(b);
});
test('include_globs change → different hash', () => {
const a = computeSourceConfigFingerprint({ include_globs: ['people/**'] });
const b = computeSourceConfigFingerprint({ include_globs: ['people/**', 'companies/**'] });
expect(a).not.toBe(b);
});
test('strategy change → different hash', () => {
const a = computeSourceConfigFingerprint({ strategy: 'markdown' });
const b = computeSourceConfigFingerprint({ strategy: 'code' });
expect(a).not.toBe(b);
});
test('strategy unset vs set differ', () => {
const unset = computeSourceConfigFingerprint({});
const set = computeSourceConfigFingerprint({ strategy: 'markdown' });
expect(unset).not.toBe(set);
});
test('add-then-remove returns to original hash (toggle is a no-op)', () => {
const original = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'] });
const added = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**', 'Photos/**'] });
const reverted = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'] });
expect(added).not.toBe(original);
expect(reverted).toBe(original);
});
test('non-walk-affecting fields are ignored (federated, unrelated keys)', () => {
const a = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'], federated: true });
const b = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'], federated: false });
const c = computeSourceConfigFingerprint({ exclude_globs: ['Templates/**'], some_unrelated_key: 'value' });
expect(a).toBe(b);
expect(a).toBe(c);
});
test('non-string strategy coerced to null (defensive)', () => {
// A hand-edited row could leave `strategy: 42` or `strategy: {}` — both
// collapse to the same shape as `strategy: undefined` so the fingerprint
// doesn't reflect a value the walker can't honor anyway.
const empty = computeSourceConfigFingerprint({});
expect(computeSourceConfigFingerprint({ strategy: 42 })).toBe(empty);
expect(computeSourceConfigFingerprint({ strategy: {} })).toBe(empty);
expect(computeSourceConfigFingerprint({ strategy: null })).toBe(empty);
});
test('defensive parsing: mixed-type glob arrays hash same as cleaned arrays', () => {
// parseGlobList drops non-string + empty entries; the fingerprint must
// reflect what the walker actually uses, not what the raw row says.
const dirty = computeSourceConfigFingerprint({
exclude_globs: ['Templates/**', 42, null, '', 'Photos/**'],
});
const clean = computeSourceConfigFingerprint({
exclude_globs: ['Templates/**', 'Photos/**'],
});
expect(dirty).toBe(clean);
});
test('empty array and missing field hash identically', () => {
const missing = computeSourceConfigFingerprint({});
const emptyArray = computeSourceConfigFingerprint({ exclude_globs: [] });
const emptyAfterClean = computeSourceConfigFingerprint({ exclude_globs: ['', '', ''] });
expect(emptyArray).toBe(missing);
expect(emptyAfterClean).toBe(missing);
});
test('non-array exclude_globs (string, object) hash same as missing', () => {
const missing = computeSourceConfigFingerprint({});
expect(computeSourceConfigFingerprint({ exclude_globs: 'Templates/**' })).toBe(missing);
expect(computeSourceConfigFingerprint({ exclude_globs: { foo: 'bar' } })).toBe(missing);
});
test('SHA-256 output shape: 64 hex characters', () => {
const fp = computeSourceConfigFingerprint({
strategy: 'markdown',
include_globs: ['people/**'],
exclude_globs: ['Templates/**', '.git/**'],
});
expect(fp).toMatch(/^[a-f0-9]{64}$/);
});
});